31#include "llvm/ADT/APFloat.h"
32#include "llvm/ADT/APInt.h"
33#include "llvm/ADT/SmallPtrSet.h"
34#include "llvm/ADT/StringExtras.h"
35#include "llvm/Analysis/ValueTracking.h"
36#include "llvm/IR/DataLayout.h"
37#include "llvm/IR/InlineAsm.h"
38#include "llvm/IR/Intrinsics.h"
39#include "llvm/IR/IntrinsicsAArch64.h"
40#include "llvm/IR/IntrinsicsAMDGPU.h"
41#include "llvm/IR/IntrinsicsARM.h"
42#include "llvm/IR/IntrinsicsBPF.h"
43#include "llvm/IR/IntrinsicsHexagon.h"
44#include "llvm/IR/IntrinsicsNVPTX.h"
45#include "llvm/IR/IntrinsicsPowerPC.h"
46#include "llvm/IR/IntrinsicsR600.h"
47#include "llvm/IR/IntrinsicsRISCV.h"
48#include "llvm/IR/IntrinsicsS390.h"
49#include "llvm/IR/IntrinsicsVE.h"
50#include "llvm/IR/IntrinsicsWebAssembly.h"
51#include "llvm/IR/IntrinsicsX86.h"
52#include "llvm/IR/MDBuilder.h"
53#include "llvm/IR/MatrixBuilder.h"
54#include "llvm/Support/ConvertUTF.h"
55#include "llvm/Support/ScopedPrinter.h"
56#include "llvm/Support/X86TargetParser.h"
60using namespace CodeGen;
65 return std::min(High, std::max(Low,
Value));
69 Align AlignmentInBytes) {
71 switch (CGF.
getLangOpts().getTrivialAutoVarInit()) {
72 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
75 case LangOptions::TrivialAutoVarInitKind::Zero:
76 Byte = CGF.
Builder.getInt8(0x00);
78 case LangOptions::TrivialAutoVarInitKind::Pattern: {
80 Byte = llvm::dyn_cast<llvm::ConstantInt>(
88 I->addAnnotationMetadata(
"auto-init");
103 static SmallDenseMap<unsigned, StringRef, 8> F128Builtins{
104 {Builtin::BI__builtin_printf,
"__printfieee128"},
105 {Builtin::BI__builtin_vsnprintf,
"__vsnprintfieee128"},
106 {Builtin::BI__builtin_vsprintf,
"__vsprintfieee128"},
107 {Builtin::BI__builtin_sprintf,
"__sprintfieee128"},
108 {Builtin::BI__builtin_snprintf,
"__snprintfieee128"},
109 {Builtin::BI__builtin_fprintf,
"__fprintfieee128"},
110 {Builtin::BI__builtin_nexttowardf128,
"__nexttowardieee128"},
116 if (FD->
hasAttr<AsmLabelAttr>())
122 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
123 F128Builtins.find(BuiltinID) != F128Builtins.end())
124 Name = F128Builtins[BuiltinID];
129 llvm::FunctionType *Ty =
132 return GetOrCreateLLVMFunction(Name, Ty, D,
false);
138 QualType T, llvm::IntegerType *IntType) {
141 if (
V->getType()->isPointerTy())
142 return CGF.
Builder.CreatePtrToInt(
V, IntType);
144 assert(
V->getType() == IntType);
149 QualType T, llvm::Type *ResultType) {
152 if (ResultType->isPointerTy())
153 return CGF.
Builder.CreateIntToPtr(
V, ResultType);
155 assert(
V->getType() == ResultType);
163 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
172 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
174 llvm::IntegerType *IntType =
177 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
179 llvm::Value *Args[2];
180 Args[0] = CGF.
Builder.CreateBitCast(DestPtr, IntPtrType);
182 llvm::Type *ValueType = Args[1]->getType();
183 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
186 Kind, Args[0], Args[1], Ordering);
196 unsigned SrcAddrSpace =
Address->
getType()->getPointerAddressSpace();
198 Address, llvm::PointerType::get(Val->getType(), SrcAddrSpace),
"cast");
200 LV.setNontemporal(
true);
209 LV.setNontemporal(
true);
214 llvm::AtomicRMWInst::BinOp Kind,
223 llvm::AtomicRMWInst::BinOp Kind,
225 Instruction::BinaryOps Op,
226 bool Invert =
false) {
234 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
236 llvm::IntegerType *IntType =
239 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
241 llvm::Value *Args[2];
243 llvm::Type *ValueType = Args[1]->getType();
244 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
245 Args[0] = CGF.
Builder.CreateBitCast(DestPtr, IntPtrType);
248 Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
253 llvm::ConstantInt::getAllOnesValue(IntType));
276 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
278 llvm::IntegerType *IntType = llvm::IntegerType::get(
280 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
283 Args[0] = CGF.
Builder.CreateBitCast(DestPtr, IntPtrType);
285 llvm::Type *ValueType = Args[1]->getType();
286 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
290 Args[0], Args[1], Args[2], llvm::AtomicOrdering::SequentiallyConsistent,
291 llvm::AtomicOrdering::SequentiallyConsistent);
294 return CGF.
Builder.CreateZExt(CGF.
Builder.CreateExtractValue(Pair, 1),
317 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
331 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
332 AtomicOrdering::Monotonic :
340 Destination, Comparand, Exchange,
341 SuccessOrdering, FailureOrdering);
342 Result->setVolatile(
true);
356 AtomicOrdering SuccessOrdering) {
363 assert(Destination->getType()->isPointerTy());
364 assert(!ExchangeHigh->getType()->isPointerTy());
365 assert(!ExchangeLow->getType()->isPointerTy());
366 assert(ComparandPtr->getType()->isPointerTy());
369 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
370 ? AtomicOrdering::Monotonic
374 llvm::Type *Int128Ty = llvm::IntegerType::get(CGF.
getLLVMContext(), 128);
375 llvm::Type *Int128PtrTy = Int128Ty->getPointerTo();
376 Destination = CGF.
Builder.CreateBitCast(Destination, Int128PtrTy);
377 Address ComparandResult(CGF.
Builder.CreateBitCast(ComparandPtr, Int128PtrTy),
381 ExchangeHigh = CGF.
Builder.CreateZExt(ExchangeHigh, Int128Ty);
382 ExchangeLow = CGF.
Builder.CreateZExt(ExchangeLow, Int128Ty);
384 CGF.
Builder.CreateShl(ExchangeHigh, llvm::ConstantInt::get(Int128Ty, 64));
385 llvm::Value *Exchange = CGF.
Builder.CreateOr(ExchangeHigh, ExchangeLow);
391 SuccessOrdering, FailureOrdering);
397 CXI->setVolatile(
true);
404 Value *Success = CGF.
Builder.CreateExtractValue(CXI, 1);
409 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
416 ConstantInt::get(IntTy, 1),
418 return CGF.
Builder.CreateAdd(
Result, ConstantInt::get(IntTy, 1));
422 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
429 ConstantInt::get(IntTy, 1),
431 return CGF.
Builder.CreateSub(
Result, ConstantInt::get(IntTy, 1));
441 Ptr = CGF.
Builder.CreateBitCast(Ptr, ITy->getPointerTo());
443 Load->setVolatile(
true);
455 Ptr = CGF.
Builder.CreateBitCast(Ptr, ITy->getPointerTo());
456 llvm::StoreInst *Store =
458 Store->setVolatile(
true);
466 const CallExpr *E,
unsigned IntrinsicID,
467 unsigned ConstrainedIntrinsicID) {
470 if (CGF.
Builder.getIsFPConstrained()) {
471 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
472 Function *F = CGF.
CGM.
getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
473 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0 });
476 return CGF.
Builder.CreateCall(F, Src0);
483 const CallExpr *E,
unsigned IntrinsicID,
484 unsigned ConstrainedIntrinsicID) {
488 if (CGF.
Builder.getIsFPConstrained()) {
489 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
490 Function *F = CGF.
CGM.
getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
491 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
494 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
501 const CallExpr *E,
unsigned IntrinsicID,
502 unsigned ConstrainedIntrinsicID) {
507 if (CGF.
Builder.getIsFPConstrained()) {
508 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
509 Function *F = CGF.
CGM.
getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
510 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
513 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
520 unsigned IntrinsicID,
521 unsigned ConstrainedIntrinsicID,
525 if (CGF.
Builder.getIsFPConstrained())
530 if (CGF.
Builder.getIsFPConstrained())
531 return CGF.
Builder.CreateConstrainedFPCall(F, Args);
533 return CGF.
Builder.CreateCall(F, Args);
539 unsigned IntrinsicID,
540 llvm::StringRef Name =
"") {
544 return CGF.
Builder.CreateCall(F, Src0, Name);
550 unsigned IntrinsicID) {
555 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
561 unsigned IntrinsicID) {
567 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
573 unsigned IntrinsicID) {
578 return CGF.
Builder.CreateCall(F, {Src0, Src1});
584 unsigned IntrinsicID,
585 unsigned ConstrainedIntrinsicID) {
589 if (CGF.
Builder.getIsFPConstrained()) {
590 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
592 {ResultType, Src0->getType()});
593 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0});
597 return CGF.
Builder.CreateCall(F, Src0);
604 llvm::CallInst *Call = CGF.
Builder.CreateCall(F,
V);
605 Call->setDoesNotAccessMemory();
614 llvm::Type *Ty =
V->getType();
615 int Width = Ty->getPrimitiveSizeInBits();
616 llvm::Type *IntTy = llvm::IntegerType::get(
C, Width);
618 if (Ty->isPPC_FP128Ty()) {
628 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
633 IntTy = llvm::IntegerType::get(
C, Width);
636 Value *Zero = llvm::Constant::getNullValue(IntTy);
637 return CGF.
Builder.CreateICmpSLT(
V, Zero);
641 const CallExpr *E, llvm::Constant *calleeValue) {
656 const llvm::Intrinsic::ID IntrinsicID,
657 llvm::Value *
X, llvm::Value *Y,
658 llvm::Value *&Carry) {
660 assert(
X->getType() == Y->getType() &&
661 "Arguments must be the same type. (Did you forget to make sure both "
662 "arguments have the same integer width?)");
665 llvm::Value *Tmp = CGF.
Builder.CreateCall(Callee, {
X, Y});
666 Carry = CGF.
Builder.CreateExtractValue(Tmp, 1);
667 return CGF.
Builder.CreateExtractValue(Tmp, 0);
671 unsigned IntrinsicID,
674 llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));
676 llvm::Instruction *Call = CGF.
Builder.CreateCall(F);
677 Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
682 struct WidthAndSignedness {
688static WidthAndSignedness
702static struct WidthAndSignedness
704 assert(Types.size() > 0 &&
"Empty list of types.");
708 for (
const auto &
Type : Types) {
717 for (
const auto &
Type : Types) {
719 if (Width < MinWidth) {
729 if (ArgValue->getType() != DestType)
731 Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data());
733 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
743 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
748 return ConstantInt::get(ResType, (
Type & 2) ? 0 : -1,
true);
752CodeGenFunction::evaluateOrEmitBuiltinObjectSize(
const Expr *E,
unsigned Type,
753 llvm::IntegerType *ResType,
754 llvm::Value *EmittedE,
758 return emitBuiltinObjectSize(E,
Type, ResType, EmittedE, IsDynamic);
759 return ConstantInt::get(ResType, ObjectSize,
true);
772CodeGenFunction::emitBuiltinObjectSize(
const Expr *E,
unsigned Type,
773 llvm::IntegerType *ResType,
774 llvm::Value *EmittedE,
bool IsDynamic) {
778 auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());
779 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
780 if (Param !=
nullptr && PS !=
nullptr &&
782 auto Iter = SizeArguments.find(Param);
783 assert(Iter != SizeArguments.end());
786 auto DIter = LocalDeclMap.find(D);
787 assert(DIter != LocalDeclMap.end());
801 assert(Ptr->getType()->isPointerTy() &&
802 "Non-pointer passed to __builtin_object_size?");
808 Value *Min = Builder.getInt1((
Type & 2) != 0);
810 Value *NullIsUnknown = Builder.getTrue();
812 return Builder.CreateCall(F, {Ptr, Min, NullIsUnknown,
Dynamic});
818 enum ActionKind : uint8_t { TestOnly, Complement, Reset, Set };
819 enum InterlockingKind : uint8_t {
828 InterlockingKind Interlocking;
831 static BitTest decodeBitTestBuiltin(
unsigned BuiltinID);
835BitTest BitTest::decodeBitTestBuiltin(
unsigned BuiltinID) {
838 case Builtin::BI_bittest:
839 return {TestOnly, Unlocked,
false};
840 case Builtin::BI_bittestandcomplement:
841 return {Complement, Unlocked,
false};
842 case Builtin::BI_bittestandreset:
843 return {Reset, Unlocked,
false};
844 case Builtin::BI_bittestandset:
845 return {Set, Unlocked,
false};
846 case Builtin::BI_interlockedbittestandreset:
847 return {Reset, Sequential,
false};
848 case Builtin::BI_interlockedbittestandset:
849 return {Set, Sequential,
false};
852 case Builtin::BI_bittest64:
853 return {TestOnly, Unlocked,
true};
854 case Builtin::BI_bittestandcomplement64:
855 return {Complement, Unlocked,
true};
856 case Builtin::BI_bittestandreset64:
857 return {Reset, Unlocked,
true};
858 case Builtin::BI_bittestandset64:
859 return {Set, Unlocked,
true};
860 case Builtin::BI_interlockedbittestandreset64:
861 return {Reset, Sequential,
true};
862 case Builtin::BI_interlockedbittestandset64:
863 return {Set, Sequential,
true};
866 case Builtin::BI_interlockedbittestandset_acq:
867 return {Set, Acquire,
false};
868 case Builtin::BI_interlockedbittestandset_rel:
869 return {Set, Release,
false};
870 case Builtin::BI_interlockedbittestandset_nf:
871 return {Set, NoFence,
false};
872 case Builtin::BI_interlockedbittestandreset_acq:
873 return {Reset, Acquire,
false};
874 case Builtin::BI_interlockedbittestandreset_rel:
875 return {Reset, Release,
false};
876 case Builtin::BI_interlockedbittestandreset_nf:
877 return {Reset, NoFence,
false};
879 llvm_unreachable(
"expected only bittest intrinsics");
884 case BitTest::TestOnly:
return '\0';
885 case BitTest::Complement:
return 'c';
886 case BitTest::Reset:
return 'r';
887 case BitTest::Set:
return 's';
889 llvm_unreachable(
"invalid action");
897 char SizeSuffix = BT.Is64Bit ?
'q' :
'l';
901 raw_svector_ostream AsmOS(
Asm);
902 if (BT.Interlocking != BitTest::Unlocked)
907 AsmOS << SizeSuffix <<
" $2, ($1)";
910 std::string Constraints =
"={@ccc},r,r,~{cc},~{memory}";
912 if (!MachineClobbers.empty()) {
914 Constraints += MachineClobbers;
916 llvm::IntegerType *IntType = llvm::IntegerType::get(
919 llvm::Type *IntPtrType = IntType->getPointerTo();
920 llvm::FunctionType *FTy =
921 llvm::FunctionType::get(CGF.
Int8Ty, {IntPtrType, IntType},
false);
923 llvm::InlineAsm *IA =
924 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
925 return CGF.
Builder.CreateCall(IA, {BitBase, BitPos});
928static llvm::AtomicOrdering
931 case BitTest::Unlocked:
return llvm::AtomicOrdering::NotAtomic;
932 case BitTest::Sequential:
return llvm::AtomicOrdering::SequentiallyConsistent;
933 case BitTest::Acquire:
return llvm::AtomicOrdering::Acquire;
934 case BitTest::Release:
return llvm::AtomicOrdering::Release;
935 case BitTest::NoFence:
return llvm::AtomicOrdering::Monotonic;
937 llvm_unreachable(
"invalid interlocking");
950 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
962 BitPos, llvm::ConstantInt::get(BitPos->getType(), 3),
"bittest.byteidx");
965 ByteIndex,
"bittest.byteaddr"),
969 llvm::ConstantInt::get(CGF.
Int8Ty, 0x7));
972 Value *Mask =
nullptr;
973 if (BT.Action != BitTest::TestOnly) {
974 Mask = CGF.
Builder.CreateShl(llvm::ConstantInt::get(CGF.
Int8Ty, 1), PosLow,
981 Value *OldByte =
nullptr;
982 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
985 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
986 if (BT.Action == BitTest::Reset) {
987 Mask = CGF.
Builder.CreateNot(Mask);
988 RMWOp = llvm::AtomicRMWInst::And;
995 Value *NewByte =
nullptr;
997 case BitTest::TestOnly:
1000 case BitTest::Complement:
1001 NewByte = CGF.
Builder.CreateXor(OldByte, Mask);
1003 case BitTest::Reset:
1004 NewByte = CGF.
Builder.CreateAnd(OldByte, CGF.
Builder.CreateNot(Mask));
1007 NewByte = CGF.
Builder.CreateOr(OldByte, Mask);
1016 Value *ShiftedByte = CGF.
Builder.CreateLShr(OldByte, PosLow,
"bittest.shr");
1018 ShiftedByte, llvm::ConstantInt::get(CGF.
Int8Ty, 1),
"bittest.res");
1027 raw_svector_ostream AsmOS(
Asm);
1028 llvm::IntegerType *RetType = CGF.
Int32Ty;
1030 switch (BuiltinID) {
1031 case clang::PPC::BI__builtin_ppc_ldarx:
1035 case clang::PPC::BI__builtin_ppc_lwarx:
1039 case clang::PPC::BI__builtin_ppc_lharx:
1043 case clang::PPC::BI__builtin_ppc_lbarx:
1048 llvm_unreachable(
"Expected only PowerPC load reserve intrinsics");
1051 AsmOS <<
"$0, ${1:y}";
1053 std::string Constraints =
"=r,*Z,~{memory}";
1055 if (!MachineClobbers.empty()) {
1057 Constraints += MachineClobbers;
1060 llvm::Type *IntPtrType = RetType->getPointerTo();
1061 llvm::FunctionType *FTy =
1062 llvm::FunctionType::get(RetType, {IntPtrType},
false);
1064 llvm::InlineAsm *IA =
1065 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1066 llvm::CallInst *CI = CGF.
Builder.CreateCall(IA, {Addr});
1068 0, Attribute::get(CGF.
getLLVMContext(), Attribute::ElementType, RetType));
1073enum class MSVCSetJmpKind {
1085 llvm::Value *Arg1 =
nullptr;
1086 llvm::Type *Arg1Ty =
nullptr;
1088 bool IsVarArg =
false;
1089 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1092 Arg1 = llvm::ConstantInt::get(CGF.
IntTy, 0);
1095 Name = SJKind == MSVCSetJmpKind::_setjmp ?
"_setjmp" :
"_setjmpex";
1098 Arg1 = CGF.
Builder.CreateCall(
1101 Arg1 = CGF.
Builder.CreateCall(
1103 llvm::ConstantInt::get(CGF.
Int32Ty, 0));
1107 llvm::Type *ArgTypes[2] = {CGF.
Int8PtrTy, Arg1Ty};
1108 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1110 llvm::Attribute::ReturnsTwice);
1112 llvm::FunctionType::get(CGF.
IntTy, ArgTypes, IsVarArg), Name,
1113 ReturnsTwiceAttr,
true);
1115 llvm::Value *Buf = CGF.
Builder.CreateBitOrPointerCast(
1117 llvm::Value *Args[] = {Buf, Arg1};
1119 CB->setAttributes(ReturnsTwiceAttr);
1170 switch (BuiltinID) {
1173 case clang::ARM::BI_BitScanForward:
1174 case clang::ARM::BI_BitScanForward64:
1175 return MSVCIntrin::_BitScanForward;
1176 case clang::ARM::BI_BitScanReverse:
1177 case clang::ARM::BI_BitScanReverse64:
1178 return MSVCIntrin::_BitScanReverse;
1179 case clang::ARM::BI_InterlockedAnd64:
1180 return MSVCIntrin::_InterlockedAnd;
1181 case clang::ARM::BI_InterlockedExchange64:
1182 return MSVCIntrin::_InterlockedExchange;
1183 case clang::ARM::BI_InterlockedExchangeAdd64:
1184 return MSVCIntrin::_InterlockedExchangeAdd;
1185 case clang::ARM::BI_InterlockedExchangeSub64:
1186 return MSVCIntrin::_InterlockedExchangeSub;
1187 case clang::ARM::BI_InterlockedOr64:
1188 return MSVCIntrin::_InterlockedOr;
1189 case clang::ARM::BI_InterlockedXor64:
1190 return MSVCIntrin::_InterlockedXor;
1191 case clang::ARM::BI_InterlockedDecrement64:
1192 return MSVCIntrin::_InterlockedDecrement;
1193 case clang::ARM::BI_InterlockedIncrement64:
1194 return MSVCIntrin::_InterlockedIncrement;
1195 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
1196 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
1197 case clang::ARM::BI_InterlockedExchangeAdd_acq:
1198 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
1199 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1200 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
1201 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
1202 case clang::ARM::BI_InterlockedExchangeAdd_rel:
1203 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
1204 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1205 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
1206 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
1207 case clang::ARM::BI_InterlockedExchangeAdd_nf:
1208 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
1209 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1210 case clang::ARM::BI_InterlockedExchange8_acq:
1211 case clang::ARM::BI_InterlockedExchange16_acq:
1212 case clang::ARM::BI_InterlockedExchange_acq:
1213 case clang::ARM::BI_InterlockedExchange64_acq:
1214 return MSVCIntrin::_InterlockedExchange_acq;
1215 case clang::ARM::BI_InterlockedExchange8_rel:
1216 case clang::ARM::BI_InterlockedExchange16_rel:
1217 case clang::ARM::BI_InterlockedExchange_rel:
1218 case clang::ARM::BI_InterlockedExchange64_rel:
1219 return MSVCIntrin::_InterlockedExchange_rel;
1220 case clang::ARM::BI_InterlockedExchange8_nf:
1221 case clang::ARM::BI_InterlockedExchange16_nf:
1222 case clang::ARM::BI_InterlockedExchange_nf:
1223 case clang::ARM::BI_InterlockedExchange64_nf:
1224 return MSVCIntrin::_InterlockedExchange_nf;
1225 case clang::ARM::BI_InterlockedCompareExchange8_acq:
1226 case clang::ARM::BI_InterlockedCompareExchange16_acq:
1227 case clang::ARM::BI_InterlockedCompareExchange_acq:
1228 case clang::ARM::BI_InterlockedCompareExchange64_acq:
1229 return MSVCIntrin::_InterlockedCompareExchange_acq;
1230 case clang::ARM::BI_InterlockedCompareExchange8_rel:
1231 case clang::ARM::BI_InterlockedCompareExchange16_rel:
1232 case clang::ARM::BI_InterlockedCompareExchange_rel:
1233 case clang::ARM::BI_InterlockedCompareExchange64_rel:
1234 return MSVCIntrin::_InterlockedCompareExchange_rel;
1235 case clang::ARM::BI_InterlockedCompareExchange8_nf:
1236 case clang::ARM::BI_InterlockedCompareExchange16_nf:
1237 case clang::ARM::BI_InterlockedCompareExchange_nf:
1238 case clang::ARM::BI_InterlockedCompareExchange64_nf:
1239 return MSVCIntrin::_InterlockedCompareExchange_nf;
1240 case clang::ARM::BI_InterlockedOr8_acq:
1241 case clang::ARM::BI_InterlockedOr16_acq:
1242 case clang::ARM::BI_InterlockedOr_acq:
1243 case clang::ARM::BI_InterlockedOr64_acq:
1244 return MSVCIntrin::_InterlockedOr_acq;
1245 case clang::ARM::BI_InterlockedOr8_rel:
1246 case clang::ARM::BI_InterlockedOr16_rel:
1247 case clang::ARM::BI_InterlockedOr_rel:
1248 case clang::ARM::BI_InterlockedOr64_rel:
1249 return MSVCIntrin::_InterlockedOr_rel;
1250 case clang::ARM::BI_InterlockedOr8_nf:
1251 case clang::ARM::BI_InterlockedOr16_nf:
1252 case clang::ARM::BI_InterlockedOr_nf:
1253 case clang::ARM::BI_InterlockedOr64_nf:
1254 return MSVCIntrin::_InterlockedOr_nf;
1255 case clang::ARM::BI_InterlockedXor8_acq:
1256 case clang::ARM::BI_InterlockedXor16_acq:
1257 case clang::ARM::BI_InterlockedXor_acq:
1258 case clang::ARM::BI_InterlockedXor64_acq:
1259 return MSVCIntrin::_InterlockedXor_acq;
1260 case clang::ARM::BI_InterlockedXor8_rel:
1261 case clang::ARM::BI_InterlockedXor16_rel:
1262 case clang::ARM::BI_InterlockedXor_rel:
1263 case clang::ARM::BI_InterlockedXor64_rel:
1264 return MSVCIntrin::_InterlockedXor_rel;
1265 case clang::ARM::BI_InterlockedXor8_nf:
1266 case clang::ARM::BI_InterlockedXor16_nf:
1267 case clang::ARM::BI_InterlockedXor_nf:
1268 case clang::ARM::BI_InterlockedXor64_nf:
1269 return MSVCIntrin::_InterlockedXor_nf;
1270 case clang::ARM::BI_InterlockedAnd8_acq:
1271 case clang::ARM::BI_InterlockedAnd16_acq:
1272 case clang::ARM::BI_InterlockedAnd_acq:
1273 case clang::ARM::BI_InterlockedAnd64_acq:
1274 return MSVCIntrin::_InterlockedAnd_acq;
1275 case clang::ARM::BI_InterlockedAnd8_rel:
1276 case clang::ARM::BI_InterlockedAnd16_rel:
1277 case clang::ARM::BI_InterlockedAnd_rel:
1278 case clang::ARM::BI_InterlockedAnd64_rel:
1279 return MSVCIntrin::_InterlockedAnd_rel;
1280 case clang::ARM::BI_InterlockedAnd8_nf:
1281 case clang::ARM::BI_InterlockedAnd16_nf:
1282 case clang::ARM::BI_InterlockedAnd_nf:
1283 case clang::ARM::BI_InterlockedAnd64_nf:
1284 return MSVCIntrin::_InterlockedAnd_nf;
1285 case clang::ARM::BI_InterlockedIncrement16_acq:
1286 case clang::ARM::BI_InterlockedIncrement_acq:
1287 case clang::ARM::BI_InterlockedIncrement64_acq:
1288 return MSVCIntrin::_InterlockedIncrement_acq;
1289 case clang::ARM::BI_InterlockedIncrement16_rel:
1290 case clang::ARM::BI_InterlockedIncrement_rel:
1291 case clang::ARM::BI_InterlockedIncrement64_rel:
1292 return MSVCIntrin::_InterlockedIncrement_rel;
1293 case clang::ARM::BI_InterlockedIncrement16_nf:
1294 case clang::ARM::BI_InterlockedIncrement_nf:
1295 case clang::ARM::BI_InterlockedIncrement64_nf:
1296 return MSVCIntrin::_InterlockedIncrement_nf;
1297 case clang::ARM::BI_InterlockedDecrement16_acq:
1298 case clang::ARM::BI_InterlockedDecrement_acq:
1299 case clang::ARM::BI_InterlockedDecrement64_acq:
1300 return MSVCIntrin::_InterlockedDecrement_acq;
1301 case clang::ARM::BI_InterlockedDecrement16_rel:
1302 case clang::ARM::BI_InterlockedDecrement_rel:
1303 case clang::ARM::BI_InterlockedDecrement64_rel:
1304 return MSVCIntrin::_InterlockedDecrement_rel;
1305 case clang::ARM::BI_InterlockedDecrement16_nf:
1306 case clang::ARM::BI_InterlockedDecrement_nf:
1307 case clang::ARM::BI_InterlockedDecrement64_nf:
1308 return MSVCIntrin::_InterlockedDecrement_nf;
1310 llvm_unreachable(
"must return from switch");
1316 switch (BuiltinID) {
1319 case clang::AArch64::BI_BitScanForward:
1320 case clang::AArch64::BI_BitScanForward64:
1321 return MSVCIntrin::_BitScanForward;
1322 case clang::AArch64::BI_BitScanReverse:
1323 case clang::AArch64::BI_BitScanReverse64:
1324 return MSVCIntrin::_BitScanReverse;
1325 case clang::AArch64::BI_InterlockedAnd64:
1326 return MSVCIntrin::_InterlockedAnd;
1327 case clang::AArch64::BI_InterlockedExchange64:
1328 return MSVCIntrin::_InterlockedExchange;
1329 case clang::AArch64::BI_InterlockedExchangeAdd64:
1330 return MSVCIntrin::_InterlockedExchangeAdd;
1331 case clang::AArch64::BI_InterlockedExchangeSub64:
1332 return MSVCIntrin::_InterlockedExchangeSub;
1333 case clang::AArch64::BI_InterlockedOr64:
1334 return MSVCIntrin::_InterlockedOr;
1335 case clang::AArch64::BI_InterlockedXor64:
1336 return MSVCIntrin::_InterlockedXor;
1337 case clang::AArch64::BI_InterlockedDecrement64:
1338 return MSVCIntrin::_InterlockedDecrement;
1339 case clang::AArch64::BI_InterlockedIncrement64:
1340 return MSVCIntrin::_InterlockedIncrement;
1341 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
1342 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
1343 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
1344 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
1345 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1346 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
1347 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
1348 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
1349 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
1350 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1351 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
1352 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
1353 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
1354 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
1355 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1356 case clang::AArch64::BI_InterlockedExchange8_acq:
1357 case clang::AArch64::BI_InterlockedExchange16_acq:
1358 case clang::AArch64::BI_InterlockedExchange_acq:
1359 case clang::AArch64::BI_InterlockedExchange64_acq:
1360 return MSVCIntrin::_InterlockedExchange_acq;
1361 case clang::AArch64::BI_InterlockedExchange8_rel:
1362 case clang::AArch64::BI_InterlockedExchange16_rel:
1363 case clang::AArch64::BI_InterlockedExchange_rel:
1364 case clang::AArch64::BI_InterlockedExchange64_rel:
1365 return MSVCIntrin::_InterlockedExchange_rel;
1366 case clang::AArch64::BI_InterlockedExchange8_nf:
1367 case clang::AArch64::BI_InterlockedExchange16_nf:
1368 case clang::AArch64::BI_InterlockedExchange_nf:
1369 case clang::AArch64::BI_InterlockedExchange64_nf:
1370 return MSVCIntrin::_InterlockedExchange_nf;
1371 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
1372 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
1373 case clang::AArch64::BI_InterlockedCompareExchange_acq:
1374 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
1375 return MSVCIntrin::_InterlockedCompareExchange_acq;
1376 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
1377 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
1378 case clang::AArch64::BI_InterlockedCompareExchange_rel:
1379 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
1380 return MSVCIntrin::_InterlockedCompareExchange_rel;
1381 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
1382 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
1383 case clang::AArch64::BI_InterlockedCompareExchange_nf:
1384 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
1385 return MSVCIntrin::_InterlockedCompareExchange_nf;
1386 case clang::AArch64::BI_InterlockedCompareExchange128:
1387 return MSVCIntrin::_InterlockedCompareExchange128;
1388 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
1389 return MSVCIntrin::_InterlockedCompareExchange128_acq;
1390 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
1391 return MSVCIntrin::_InterlockedCompareExchange128_nf;
1392 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
1393 return MSVCIntrin::_InterlockedCompareExchange128_rel;
1394 case clang::AArch64::BI_InterlockedOr8_acq:
1395 case clang::AArch64::BI_InterlockedOr16_acq:
1396 case clang::AArch64::BI_InterlockedOr_acq:
1397 case clang::AArch64::BI_InterlockedOr64_acq:
1398 return MSVCIntrin::_InterlockedOr_acq;
1399 case clang::AArch64::BI_InterlockedOr8_rel:
1400 case clang::AArch64::BI_InterlockedOr16_rel:
1401 case clang::AArch64::BI_InterlockedOr_rel:
1402 case clang::AArch64::BI_InterlockedOr64_rel:
1403 return MSVCIntrin::_InterlockedOr_rel;
1404 case clang::AArch64::BI_InterlockedOr8_nf:
1405 case clang::AArch64::BI_InterlockedOr16_nf:
1406 case clang::AArch64::BI_InterlockedOr_nf:
1407 case clang::AArch64::BI_InterlockedOr64_nf:
1408 return MSVCIntrin::_InterlockedOr_nf;
1409 case clang::AArch64::BI_InterlockedXor8_acq:
1410 case clang::AArch64::BI_InterlockedXor16_acq:
1411 case clang::AArch64::BI_InterlockedXor_acq:
1412 case clang::AArch64::BI_InterlockedXor64_acq:
1413 return MSVCIntrin::_InterlockedXor_acq;
1414 case clang::AArch64::BI_InterlockedXor8_rel:
1415 case clang::AArch64::BI_InterlockedXor16_rel:
1416 case clang::AArch64::BI_InterlockedXor_rel:
1417 case clang::AArch64::BI_InterlockedXor64_rel:
1418 return MSVCIntrin::_InterlockedXor_rel;
1419 case clang::AArch64::BI_InterlockedXor8_nf:
1420 case clang::AArch64::BI_InterlockedXor16_nf:
1421 case clang::AArch64::BI_InterlockedXor_nf:
1422 case clang::AArch64::BI_InterlockedXor64_nf:
1423 return MSVCIntrin::_InterlockedXor_nf;
1424 case clang::AArch64::BI_InterlockedAnd8_acq:
1425 case clang::AArch64::BI_InterlockedAnd16_acq:
1426 case clang::AArch64::BI_InterlockedAnd_acq:
1427 case clang::AArch64::BI_InterlockedAnd64_acq:
1428 return MSVCIntrin::_InterlockedAnd_acq;
1429 case clang::AArch64::BI_InterlockedAnd8_rel:
1430 case clang::AArch64::BI_InterlockedAnd16_rel:
1431 case clang::AArch64::BI_InterlockedAnd_rel:
1432 case clang::AArch64::BI_InterlockedAnd64_rel:
1433 return MSVCIntrin::_InterlockedAnd_rel;
1434 case clang::AArch64::BI_InterlockedAnd8_nf:
1435 case clang::AArch64::BI_InterlockedAnd16_nf:
1436 case clang::AArch64::BI_InterlockedAnd_nf:
1437 case clang::AArch64::BI_InterlockedAnd64_nf:
1438 return MSVCIntrin::_InterlockedAnd_nf;
1439 case clang::AArch64::BI_InterlockedIncrement16_acq:
1440 case clang::AArch64::BI_InterlockedIncrement_acq:
1441 case clang::AArch64::BI_InterlockedIncrement64_acq:
1442 return MSVCIntrin::_InterlockedIncrement_acq;
1443 case clang::AArch64::BI_InterlockedIncrement16_rel:
1444 case clang::AArch64::BI_InterlockedIncrement_rel:
1445 case clang::AArch64::BI_InterlockedIncrement64_rel:
1446 return MSVCIntrin::_InterlockedIncrement_rel;
1447 case clang::AArch64::BI_InterlockedIncrement16_nf:
1448 case clang::AArch64::BI_InterlockedIncrement_nf:
1449 case clang::AArch64::BI_InterlockedIncrement64_nf:
1450 return MSVCIntrin::_InterlockedIncrement_nf;
1451 case clang::AArch64::BI_InterlockedDecrement16_acq:
1452 case clang::AArch64::BI_InterlockedDecrement_acq:
1453 case clang::AArch64::BI_InterlockedDecrement64_acq:
1454 return MSVCIntrin::_InterlockedDecrement_acq;
1455 case clang::AArch64::BI_InterlockedDecrement16_rel:
1456 case clang::AArch64::BI_InterlockedDecrement_rel:
1457 case clang::AArch64::BI_InterlockedDecrement64_rel:
1458 return MSVCIntrin::_InterlockedDecrement_rel;
1459 case clang::AArch64::BI_InterlockedDecrement16_nf:
1460 case clang::AArch64::BI_InterlockedDecrement_nf:
1461 case clang::AArch64::BI_InterlockedDecrement64_nf:
1462 return MSVCIntrin::_InterlockedDecrement_nf;
1464 llvm_unreachable(
"must return from switch");
1470 switch (BuiltinID) {
1473 case clang::X86::BI_BitScanForward:
1474 case clang::X86::BI_BitScanForward64:
1475 return MSVCIntrin::_BitScanForward;
1476 case clang::X86::BI_BitScanReverse:
1477 case clang::X86::BI_BitScanReverse64:
1478 return MSVCIntrin::_BitScanReverse;
1479 case clang::X86::BI_InterlockedAnd64:
1480 return MSVCIntrin::_InterlockedAnd;
1481 case clang::X86::BI_InterlockedCompareExchange128:
1482 return MSVCIntrin::_InterlockedCompareExchange128;
1483 case clang::X86::BI_InterlockedExchange64:
1484 return MSVCIntrin::_InterlockedExchange;
1485 case clang::X86::BI_InterlockedExchangeAdd64:
1486 return MSVCIntrin::_InterlockedExchangeAdd;
1487 case clang::X86::BI_InterlockedExchangeSub64:
1488 return MSVCIntrin::_InterlockedExchangeSub;
1489 case clang::X86::BI_InterlockedOr64:
1490 return MSVCIntrin::_InterlockedOr;
1491 case clang::X86::BI_InterlockedXor64:
1492 return MSVCIntrin::_InterlockedXor;
1493 case clang::X86::BI_InterlockedDecrement64:
1494 return MSVCIntrin::_InterlockedDecrement;
1495 case clang::X86::BI_InterlockedIncrement64:
1496 return MSVCIntrin::_InterlockedIncrement;
1498 llvm_unreachable(
"must return from switch");
1504 switch (BuiltinID) {
1505 case MSVCIntrin::_BitScanForward:
1506 case MSVCIntrin::_BitScanReverse: {
1510 llvm::Type *ArgType = ArgValue->getType();
1511 llvm::Type *IndexType = IndexAddress.getElementType();
1514 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
1515 Value *ResZero = llvm::Constant::getNullValue(ResultType);
1516 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
1518 BasicBlock *
Begin = Builder.GetInsertBlock();
1520 Builder.SetInsertPoint(End);
1521 PHINode *
Result = Builder.CreatePHI(ResultType, 2,
"bitscan_result");
1523 Builder.SetInsertPoint(
Begin);
1524 Value *IsZero = Builder.CreateICmpEQ(ArgValue, ArgZero);
1526 Builder.CreateCondBr(IsZero, End, NotZero);
1529 Builder.SetInsertPoint(NotZero);
1531 if (BuiltinID == MSVCIntrin::_BitScanForward) {
1533 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
1534 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType,
false);
1535 Builder.CreateStore(ZeroCount, IndexAddress,
false);
1538 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
1541 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
1542 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType,
false);
1543 Value *Index = Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
1544 Builder.CreateStore(Index, IndexAddress,
false);
1546 Builder.CreateBr(End);
1547 Result->addIncoming(ResOne, NotZero);
1549 Builder.SetInsertPoint(End);
1552 case MSVCIntrin::_InterlockedAnd:
1554 case MSVCIntrin::_InterlockedExchange:
1556 case MSVCIntrin::_InterlockedExchangeAdd:
1558 case MSVCIntrin::_InterlockedExchangeSub:
1560 case MSVCIntrin::_InterlockedOr:
1562 case MSVCIntrin::_InterlockedXor:
1564 case MSVCIntrin::_InterlockedExchangeAdd_acq:
1566 AtomicOrdering::Acquire);
1567 case MSVCIntrin::_InterlockedExchangeAdd_rel:
1569 AtomicOrdering::Release);
1570 case MSVCIntrin::_InterlockedExchangeAdd_nf:
1572 AtomicOrdering::Monotonic);
1573 case MSVCIntrin::_InterlockedExchange_acq:
1575 AtomicOrdering::Acquire);
1576 case MSVCIntrin::_InterlockedExchange_rel:
1578 AtomicOrdering::Release);
1579 case MSVCIntrin::_InterlockedExchange_nf:
1581 AtomicOrdering::Monotonic);
1582 case MSVCIntrin::_InterlockedCompareExchange_acq:
1584 case MSVCIntrin::_InterlockedCompareExchange_rel:
1586 case MSVCIntrin::_InterlockedCompareExchange_nf:
1588 case MSVCIntrin::_InterlockedCompareExchange128:
1590 *
this, E, AtomicOrdering::SequentiallyConsistent);
1591 case MSVCIntrin::_InterlockedCompareExchange128_acq:
1593 case MSVCIntrin::_InterlockedCompareExchange128_rel:
1595 case MSVCIntrin::_InterlockedCompareExchange128_nf:
1597 case MSVCIntrin::_InterlockedOr_acq:
1599 AtomicOrdering::Acquire);
1600 case MSVCIntrin::_InterlockedOr_rel:
1602 AtomicOrdering::Release);
1603 case MSVCIntrin::_InterlockedOr_nf:
1605 AtomicOrdering::Monotonic);
1606 case MSVCIntrin::_InterlockedXor_acq:
1608 AtomicOrdering::Acquire);
1609 case MSVCIntrin::_InterlockedXor_rel:
1611 AtomicOrdering::Release);
1612 case MSVCIntrin::_InterlockedXor_nf:
1614 AtomicOrdering::Monotonic);
1615 case MSVCIntrin::_InterlockedAnd_acq:
1617 AtomicOrdering::Acquire);
1618 case MSVCIntrin::_InterlockedAnd_rel:
1620 AtomicOrdering::Release);
1621 case MSVCIntrin::_InterlockedAnd_nf:
1623 AtomicOrdering::Monotonic);
1624 case MSVCIntrin::_InterlockedIncrement_acq:
1626 case MSVCIntrin::_InterlockedIncrement_rel:
1628 case MSVCIntrin::_InterlockedIncrement_nf:
1630 case MSVCIntrin::_InterlockedDecrement_acq:
1632 case MSVCIntrin::_InterlockedDecrement_rel:
1634 case MSVCIntrin::_InterlockedDecrement_nf:
1637 case MSVCIntrin::_InterlockedDecrement:
1639 case MSVCIntrin::_InterlockedIncrement:
1642 case MSVCIntrin::__fastfail: {
1647 StringRef
Asm, Constraints;
1652 case llvm::Triple::x86:
1653 case llvm::Triple::x86_64:
1655 Constraints =
"{cx}";
1657 case llvm::Triple::thumb:
1659 Constraints =
"{r0}";
1661 case llvm::Triple::aarch64:
1662 Asm =
"brk #0xF003";
1663 Constraints =
"{w0}";
1665 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy, {
Int32Ty},
false);
1666 llvm::InlineAsm *IA =
1667 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1668 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
1670 llvm::Attribute::NoReturn);
1672 CI->setAttributes(NoReturnAttr);
1676 llvm_unreachable(
"Incorrect MSVC intrinsic!");
1682 CallObjCArcUse(llvm::Value *
object) :
object(
object) {}
1692 BuiltinCheckKind Kind) {
1694 &&
"Unsupported builtin check kind");
1700 SanitizerScope SanScope(
this);
1702 ArgValue, llvm::Constant::getNullValue(ArgValue->getType()));
1704 SanitizerHandler::InvalidBuiltin,
1706 llvm::ConstantInt::get(Builder.getInt8Ty(), Kind)},
1713 QualType UnsignedTy =
C.getIntTypeForBitwidth(Size * 8,
false);
1724 raw_svector_ostream
OS(Name);
1725 OS <<
"__os_log_helper";
1729 for (
const auto &Item : Layout.
Items)
1730 OS <<
"_" <<
int(Item.getSizeByte()) <<
"_"
1731 <<
int(Item.getDescriptorByte());
1734 if (llvm::Function *F =
CGM.
getModule().getFunction(Name))
1744 for (
unsigned int I = 0, E = Layout.
Items.size(); I < E; ++I) {
1745 char Size = Layout.
Items[I].getSizeByte();
1752 &Ctx.
Idents.
get(std::string(
"arg") + llvm::to_string(I)), ArgTy,
1754 ArgTys.emplace_back(ArgTy);
1765 llvm::Function *Fn = llvm::Function::Create(
1766 FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &
CGM.
getModule());
1767 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
1770 Fn->setDoesNotThrow();
1774 Fn->addFnAttr(llvm::Attribute::NoInline);
1787 Builder.CreateConstByteGEP(BufAddr,
Offset++,
"summary"));
1789 Builder.CreateConstByteGEP(BufAddr,
Offset++,
"numArgs"));
1792 for (
const auto &Item : Layout.
Items) {
1793 Builder.CreateStore(
1794 Builder.getInt8(Item.getDescriptorByte()),
1795 Builder.CreateConstByteGEP(BufAddr,
Offset++,
"argDescriptor"));
1796 Builder.CreateStore(
1797 Builder.getInt8(Item.getSizeByte()),
1798 Builder.CreateConstByteGEP(BufAddr,
Offset++,
"argSize"));
1801 if (!
Size.getQuantity())
1805 Address Addr = Builder.CreateConstByteGEP(BufAddr,
Offset,
"argData");
1807 Builder.CreateElementBitCast(Addr, Arg.
getElementType(),
"argDataCast");
1808 Builder.CreateStore(Builder.CreateLoad(Arg), Addr);
1820 "__builtin_os_log_format takes at least 2 arguments");
1831 for (
const auto &Item : Layout.
Items) {
1832 int Size = Item.getSizeByte();
1836 llvm::Value *ArgVal;
1840 for (
unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
1841 Val |= ((
uint64_t)Item.getMaskType()[I]) << I * 8;
1842 ArgVal = llvm::Constant::getIntegerValue(
Int64Ty, llvm::APInt(64, Val));
1843 }
else if (
const Expr *TheExpr = Item.getExpr()) {
1849 auto LifetimeExtendObject = [&](
const Expr *E) {
1857 if (isa<CallExpr>(E) || isa<ObjCMessageExpr>(E))
1862 if (TheExpr->getType()->isObjCRetainableType() &&
1863 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
1865 "Only scalar can be a ObjC retainable type");
1866 if (!isa<Constant>(ArgVal)) {
1872 Builder.CreateStore(ArgVal, Addr);
1884 ArgVal = Builder.getInt32(Item.getConstValue().getQuantity());
1887 unsigned ArgValSize =
1891 ArgVal = Builder.CreateBitOrPointerCast(ArgVal,
IntTy);
1894 ArgVal = Builder.CreateZExtOrBitCast(ArgVal,
ConvertType(ArgTy));
1907 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
1908 WidthAndSignedness ResultInfo) {
1909 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
1910 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
1911 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
1916 const clang::Expr *Op2, WidthAndSignedness Op2Info,
1918 WidthAndSignedness ResultInfo) {
1920 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
1921 "Cannot specialize this multiply");
1926 llvm::Value *HasOverflow;
1928 CGF, llvm::Intrinsic::umul_with_overflow, V1, V2, HasOverflow);
1933 auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width);
1934 llvm::Value *IntMaxValue = llvm::ConstantInt::get(
Result->getType(), IntMax);
1936 llvm::Value *IntMaxOverflow = CGF.
Builder.CreateICmpUGT(
Result, IntMaxValue);
1937 HasOverflow = CGF.
Builder.CreateOr(HasOverflow, IntMaxOverflow);
1949 WidthAndSignedness Op1Info,
1950 WidthAndSignedness Op2Info,
1951 WidthAndSignedness ResultInfo) {
1952 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
1953 std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
1954 Op1Info.Signed != Op2Info.Signed;
1961 WidthAndSignedness Op1Info,
const clang::Expr *Op2,
1962 WidthAndSignedness Op2Info,
1964 WidthAndSignedness ResultInfo) {
1966 Op2Info, ResultInfo) &&
1967 "Not a mixed-sign multipliction we can specialize");
1970 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
1971 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
1974 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
1975 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
1978 if (SignedOpWidth < UnsignedOpWidth)
1980 if (UnsignedOpWidth < SignedOpWidth)
1983 llvm::Type *OpTy =
Signed->getType();
1984 llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
1987 unsigned OpWidth = std::max(Op1Info.Width, Op2Info.Width);
1990 llvm::Value *IsNegative = CGF.
Builder.CreateICmpSLT(
Signed, Zero);
1991 llvm::Value *AbsOfNegative = CGF.
Builder.CreateSub(Zero,
Signed);
1992 llvm::Value *AbsSigned =
1993 CGF.
Builder.CreateSelect(IsNegative, AbsOfNegative,
Signed);
1996 llvm::Value *UnsignedOverflow;
1997 llvm::Value *UnsignedResult =
2001 llvm::Value *Overflow, *
Result;
2002 if (ResultInfo.Signed) {
2006 llvm::APInt::getSignedMaxValue(ResultInfo.Width).zext(OpWidth);
2007 llvm::Value *MaxResult =
2008 CGF.
Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
2009 CGF.
Builder.CreateZExt(IsNegative, OpTy));
2010 llvm::Value *SignedOverflow =
2011 CGF.
Builder.CreateICmpUGT(UnsignedResult, MaxResult);
2012 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, SignedOverflow);
2015 llvm::Value *NegativeResult = CGF.
Builder.CreateNeg(UnsignedResult);
2016 llvm::Value *SignedResult =
2017 CGF.
Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
2021 llvm::Value *Underflow = CGF.
Builder.CreateAnd(
2022 IsNegative, CGF.
Builder.CreateIsNotNull(UnsignedResult));
2023 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, Underflow);
2024 if (ResultInfo.Width < OpWidth) {
2026 llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
2027 llvm::Value *TruncOverflow = CGF.
Builder.CreateICmpUGT(
2028 UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
2029 Overflow = CGF.
Builder.CreateOr(Overflow, TruncOverflow);
2034 IsNegative, CGF.
Builder.CreateNeg(UnsignedResult), UnsignedResult);
2038 assert(Overflow &&
Result &&
"Missing overflow or result");
2049 llvm::SmallPtrSetImpl<const Decl *> &Seen) {
2058 if (!Seen.insert(Record).second)
2061 assert(Record->hasDefinition() &&
2062 "Incomplete types should already be diagnosed");
2064 if (Record->isDynamicClass())
2089 llvm::Type *Ty = Src->getType();
2090 ShiftAmt = Builder.CreateIntCast(ShiftAmt, Ty,
false);
2093 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2095 return RValue::get(Builder.CreateCall(F, { Src, Src, ShiftAmt }));
2100 switch (BuiltinID) {
2101#define MUTATE_LDBL(func) \
2102 case Builtin::BI__builtin_##func##l: \
2103 return Builtin::BI__builtin_##func##f128;
2178 !
Result.hasSideEffects()) {
2182 if (
Result.Val.isFloat())
2191 if (
getTarget().getTriple().isPPC64() &&
2192 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2199 const unsigned BuiltinIDIfNoAsmLabel =
2200 FD->
hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2207 if (FD->
hasAttr<ConstAttr>()) {
2208 switch (BuiltinIDIfNoAsmLabel) {
2209 case Builtin::BIceil:
2210 case Builtin::BIceilf:
2211 case Builtin::BIceill:
2212 case Builtin::BI__builtin_ceil:
2213 case Builtin::BI__builtin_ceilf:
2214 case Builtin::BI__builtin_ceilf16:
2215 case Builtin::BI__builtin_ceill:
2216 case Builtin::BI__builtin_ceilf128:
2219 Intrinsic::experimental_constrained_ceil));
2221 case Builtin::BIcopysign:
2222 case Builtin::BIcopysignf:
2223 case Builtin::BIcopysignl:
2224 case Builtin::BI__builtin_copysign:
2225 case Builtin::BI__builtin_copysignf:
2226 case Builtin::BI__builtin_copysignf16:
2227 case Builtin::BI__builtin_copysignl:
2228 case Builtin::BI__builtin_copysignf128:
2231 case Builtin::BIcos:
2232 case Builtin::BIcosf:
2233 case Builtin::BIcosl:
2234 case Builtin::BI__builtin_cos:
2235 case Builtin::BI__builtin_cosf:
2236 case Builtin::BI__builtin_cosf16:
2237 case Builtin::BI__builtin_cosl:
2238 case Builtin::BI__builtin_cosf128:
2241 Intrinsic::experimental_constrained_cos));
2243 case Builtin::BIexp:
2244 case Builtin::BIexpf:
2245 case Builtin::BIexpl:
2246 case Builtin::BI__builtin_exp:
2247 case Builtin::BI__builtin_expf:
2248 case Builtin::BI__builtin_expf16:
2249 case Builtin::BI__builtin_expl:
2250 case Builtin::BI__builtin_expf128:
2253 Intrinsic::experimental_constrained_exp));
2255 case Builtin::BIexp2:
2256 case Builtin::BIexp2f:
2257 case Builtin::BIexp2l:
2258 case Builtin::BI__builtin_exp2:
2259 case Builtin::BI__builtin_exp2f:
2260 case Builtin::BI__builtin_exp2f16:
2261 case Builtin::BI__builtin_exp2l:
2262 case Builtin::BI__builtin_exp2f128:
2265 Intrinsic::experimental_constrained_exp2));
2267 case Builtin::BIfabs:
2268 case Builtin::BIfabsf:
2269 case Builtin::BIfabsl:
2270 case Builtin::BI__builtin_fabs:
2271 case Builtin::BI__builtin_fabsf:
2272 case Builtin::BI__builtin_fabsf16:
2273 case Builtin::BI__builtin_fabsl:
2274 case Builtin::BI__builtin_fabsf128:
2277 case Builtin::BIfloor:
2278 case Builtin::BIfloorf:
2279 case Builtin::BIfloorl:
2280 case Builtin::BI__builtin_floor:
2281 case Builtin::BI__builtin_floorf:
2282 case Builtin::BI__builtin_floorf16:
2283 case Builtin::BI__builtin_floorl:
2284 case Builtin::BI__builtin_floorf128:
2287 Intrinsic::experimental_constrained_floor));
2289 case Builtin::BIfma:
2290 case Builtin::BIfmaf:
2291 case Builtin::BIfmal:
2292 case Builtin::BI__builtin_fma:
2293 case Builtin::BI__builtin_fmaf:
2294 case Builtin::BI__builtin_fmaf16:
2295 case Builtin::BI__builtin_fmal:
2296 case Builtin::BI__builtin_fmaf128:
2299 Intrinsic::experimental_constrained_fma));
2301 case Builtin::BIfmax:
2302 case Builtin::BIfmaxf:
2303 case Builtin::BIfmaxl:
2304 case Builtin::BI__builtin_fmax:
2305 case Builtin::BI__builtin_fmaxf:
2306 case Builtin::BI__builtin_fmaxf16:
2307 case Builtin::BI__builtin_fmaxl:
2308 case Builtin::BI__builtin_fmaxf128:
2311 Intrinsic::experimental_constrained_maxnum));
2313 case Builtin::BIfmin:
2314 case Builtin::BIfminf:
2315 case Builtin::BIfminl:
2316 case Builtin::BI__builtin_fmin:
2317 case Builtin::BI__builtin_fminf:
2318 case Builtin::BI__builtin_fminf16:
2319 case Builtin::BI__builtin_fminl:
2320 case Builtin::BI__builtin_fminf128:
2323 Intrinsic::experimental_constrained_minnum));
2327 case Builtin::BIfmod:
2328 case Builtin::BIfmodf:
2329 case Builtin::BIfmodl:
2330 case Builtin::BI__builtin_fmod:
2331 case Builtin::BI__builtin_fmodf:
2332 case Builtin::BI__builtin_fmodf16:
2333 case Builtin::BI__builtin_fmodl:
2334 case Builtin::BI__builtin_fmodf128: {
2335 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
2338 return RValue::get(Builder.CreateFRem(Arg1, Arg2,
"fmod"));
2341 case Builtin::BIlog:
2342 case Builtin::BIlogf:
2343 case Builtin::BIlogl:
2344 case Builtin::BI__builtin_log:
2345 case Builtin::BI__builtin_logf:
2346 case Builtin::BI__builtin_logf16:
2347 case Builtin::BI__builtin_logl:
2348 case Builtin::BI__builtin_logf128:
2351 Intrinsic::experimental_constrained_log));
2353 case Builtin::BIlog10:
2354 case Builtin::BIlog10f:
2355 case Builtin::BIlog10l:
2356 case Builtin::BI__builtin_log10:
2357 case Builtin::BI__builtin_log10f:
2358 case Builtin::BI__builtin_log10f16:
2359 case Builtin::BI__builtin_log10l:
2360 case Builtin::BI__builtin_log10f128:
2363 Intrinsic::experimental_constrained_log10));
2365 case Builtin::BIlog2:
2366 case Builtin::BIlog2f:
2367 case Builtin::BIlog2l:
2368 case Builtin::BI__builtin_log2:
2369 case Builtin::BI__builtin_log2f:
2370 case Builtin::BI__builtin_log2f16:
2371 case Builtin::BI__builtin_log2l:
2372 case Builtin::BI__builtin_log2f128:
2375 Intrinsic::experimental_constrained_log2));
2377 case Builtin::BInearbyint:
2378 case Builtin::BInearbyintf:
2379 case Builtin::BInearbyintl:
2380 case Builtin::BI__builtin_nearbyint:
2381 case Builtin::BI__builtin_nearbyintf:
2382 case Builtin::BI__builtin_nearbyintl:
2383 case Builtin::BI__builtin_nearbyintf128:
2385 Intrinsic::nearbyint,
2386 Intrinsic::experimental_constrained_nearbyint));
2388 case Builtin::BIpow:
2389 case Builtin::BIpowf:
2390 case Builtin::BIpowl:
2391 case Builtin::BI__builtin_pow:
2392 case Builtin::BI__builtin_powf:
2393 case Builtin::BI__builtin_powf16:
2394 case Builtin::BI__builtin_powl:
2395 case Builtin::BI__builtin_powf128:
2398 Intrinsic::experimental_constrained_pow));
2400 case Builtin::BIrint:
2401 case Builtin::BIrintf:
2402 case Builtin::BIrintl:
2403 case Builtin::BI__builtin_rint:
2404 case Builtin::BI__builtin_rintf:
2405 case Builtin::BI__builtin_rintf16:
2406 case Builtin::BI__builtin_rintl:
2407 case Builtin::BI__builtin_rintf128:
2410 Intrinsic::experimental_constrained_rint));
2412 case Builtin::BIround:
2413 case Builtin::BIroundf:
2414 case Builtin::BIroundl:
2415 case Builtin::BI__builtin_round:
2416 case Builtin::BI__builtin_roundf:
2417 case Builtin::BI__builtin_roundf16:
2418 case Builtin::BI__builtin_roundl:
2419 case Builtin::BI__builtin_roundf128:
2422 Intrinsic::experimental_constrained_round));
2424 case Builtin::BIsin:
2425 case Builtin::BIsinf:
2426 case Builtin::BIsinl:
2427 case Builtin::BI__builtin_sin:
2428 case Builtin::BI__builtin_sinf:
2429 case Builtin::BI__builtin_sinf16:
2430 case Builtin::BI__builtin_sinl:
2431 case Builtin::BI__builtin_sinf128:
2434 Intrinsic::experimental_constrained_sin));
2436 case Builtin::BIsqrt:
2437 case Builtin::BIsqrtf:
2438 case Builtin::BIsqrtl:
2439 case Builtin::BI__builtin_sqrt:
2440 case Builtin::BI__builtin_sqrtf:
2441 case Builtin::BI__builtin_sqrtf16:
2442 case Builtin::BI__builtin_sqrtl:
2443 case Builtin::BI__builtin_sqrtf128:
2446 Intrinsic::experimental_constrained_sqrt));
2448 case Builtin::BItrunc:
2449 case Builtin::BItruncf:
2450 case Builtin::BItruncl:
2451 case Builtin::BI__builtin_trunc:
2452 case Builtin::BI__builtin_truncf:
2453 case Builtin::BI__builtin_truncf16:
2454 case Builtin::BI__builtin_truncl:
2455 case Builtin::BI__builtin_truncf128:
2458 Intrinsic::experimental_constrained_trunc));
2460 case Builtin::BIlround:
2461 case Builtin::BIlroundf:
2462 case Builtin::BIlroundl:
2463 case Builtin::BI__builtin_lround:
2464 case Builtin::BI__builtin_lroundf:
2465 case Builtin::BI__builtin_lroundl:
2466 case Builtin::BI__builtin_lroundf128:
2468 *
this, E, Intrinsic::lround,
2469 Intrinsic::experimental_constrained_lround));
2471 case Builtin::BIllround:
2472 case Builtin::BIllroundf:
2473 case Builtin::BIllroundl:
2474 case Builtin::BI__builtin_llround:
2475 case Builtin::BI__builtin_llroundf:
2476 case Builtin::BI__builtin_llroundl:
2477 case Builtin::BI__builtin_llroundf128:
2479 *
this, E, Intrinsic::llround,
2480 Intrinsic::experimental_constrained_llround));
2482 case Builtin::BIlrint:
2483 case Builtin::BIlrintf:
2484 case Builtin::BIlrintl:
2485 case Builtin::BI__builtin_lrint:
2486 case Builtin::BI__builtin_lrintf:
2487 case Builtin::BI__builtin_lrintl:
2488 case Builtin::BI__builtin_lrintf128:
2490 *
this, E, Intrinsic::lrint,
2491 Intrinsic::experimental_constrained_lrint));
2493 case Builtin::BIllrint:
2494 case Builtin::BIllrintf:
2495 case Builtin::BIllrintl:
2496 case Builtin::BI__builtin_llrint:
2497 case Builtin::BI__builtin_llrintf:
2498 case Builtin::BI__builtin_llrintl:
2499 case Builtin::BI__builtin_llrintf128:
2501 *
this, E, Intrinsic::llrint,
2502 Intrinsic::experimental_constrained_llrint));
2509 switch (BuiltinIDIfNoAsmLabel) {
2511 case Builtin::BI__builtin___CFStringMakeConstantString:
2512 case Builtin::BI__builtin___NSStringMakeConstantString:
2514 case Builtin::BI__builtin_stdarg_start:
2515 case Builtin::BI__builtin_va_start:
2516 case Builtin::BI__va_start:
2517 case Builtin::BI__builtin_va_end:
2522 BuiltinID != Builtin::BI__builtin_va_end));
2523 case Builtin::BI__builtin_va_copy: {
2529 DstPtr = Builder.CreateBitCast(DstPtr,
Type);
2530 SrcPtr = Builder.CreateBitCast(SrcPtr,
Type);
2534 case Builtin::BI__builtin_abs:
2535 case Builtin::BI__builtin_labs:
2536 case Builtin::BI__builtin_llabs: {
2540 Value *NegOp = Builder.CreateNSWNeg(ArgValue,
"neg");
2541 Constant *
Zero = llvm::Constant::getNullValue(ArgValue->getType());
2542 Value *CmpResult = Builder.CreateICmpSLT(ArgValue, Zero,
"abscond");
2543 Value *
Result = Builder.CreateSelect(CmpResult, NegOp, ArgValue,
"abs");
2546 case Builtin::BI__builtin_complex: {
2551 case Builtin::BI__builtin_conj:
2552 case Builtin::BI__builtin_conjf:
2553 case Builtin::BI__builtin_conjl:
2554 case Builtin::BIconj:
2555 case Builtin::BIconjf:
2556 case Builtin::BIconjl: {
2558 Value *Real = ComplexVal.first;
2559 Value *Imag = ComplexVal.second;
2560 Imag = Builder.CreateFNeg(Imag,
"neg");
2563 case Builtin::BI__builtin_creal:
2564 case Builtin::BI__builtin_crealf:
2565 case Builtin::BI__builtin_creall:
2566 case Builtin::BIcreal:
2567 case Builtin::BIcrealf:
2568 case Builtin::BIcreall: {
2573 case Builtin::BI__builtin_preserve_access_index: {
2594 case Builtin::BI__builtin_cimag:
2595 case Builtin::BI__builtin_cimagf:
2596 case Builtin::BI__builtin_cimagl:
2597 case Builtin::BIcimag:
2598 case Builtin::BIcimagf:
2599 case Builtin::BIcimagl: {
2604 case Builtin::BI__builtin_clrsb:
2605 case Builtin::BI__builtin_clrsbl:
2606 case Builtin::BI__builtin_clrsbll: {
2610 llvm::Type *ArgType = ArgValue->getType();
2614 Value *
Zero = llvm::Constant::getNullValue(ArgType);
2615 Value *IsNeg = Builder.CreateICmpSLT(ArgValue, Zero,
"isneg");
2616 Value *Inverse = Builder.CreateNot(ArgValue,
"not");
2617 Value *Tmp = Builder.CreateSelect(IsNeg, Inverse, ArgValue);
2618 Value *Ctlz = Builder.CreateCall(F, {Tmp, Builder.getFalse()});
2619 Value *
Result = Builder.CreateSub(Ctlz, llvm::ConstantInt::get(ArgType, 1));
2620 Result = Builder.CreateIntCast(
Result, ResultType,
true,
2624 case Builtin::BI__builtin_ctzs:
2625 case Builtin::BI__builtin_ctz:
2626 case Builtin::BI__builtin_ctzl:
2627 case Builtin::BI__builtin_ctzll: {
2630 llvm::Type *ArgType = ArgValue->getType();
2634 Value *ZeroUndef = Builder.getInt1(
getTarget().isCLZForZeroUndef());
2635 Value *
Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
2636 if (
Result->getType() != ResultType)
2637 Result = Builder.CreateIntCast(
Result, ResultType,
true,
2641 case Builtin::BI__builtin_clzs:
2642 case Builtin::BI__builtin_clz:
2643 case Builtin::BI__builtin_clzl:
2644 case Builtin::BI__builtin_clzll: {
2647 llvm::Type *ArgType = ArgValue->getType();
2651 Value *ZeroUndef = Builder.getInt1(
getTarget().isCLZForZeroUndef());
2652 Value *
Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
2653 if (
Result->getType() != ResultType)
2654 Result = Builder.CreateIntCast(
Result, ResultType,
true,
2658 case Builtin::BI__builtin_ffs:
2659 case Builtin::BI__builtin_ffsl:
2660 case Builtin::BI__builtin_ffsll: {
2664 llvm::Type *ArgType = ArgValue->getType();
2669 Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
2670 llvm::ConstantInt::get(ArgType, 1));
2671 Value *
Zero = llvm::Constant::getNullValue(ArgType);
2672 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
2673 Value *
Result = Builder.CreateSelect(IsZero, Zero, Tmp,
"ffs");
2674 if (
Result->getType() != ResultType)
2675 Result = Builder.CreateIntCast(
Result, ResultType,
true,
2679 case Builtin::BI__builtin_parity:
2680 case Builtin::BI__builtin_parityl:
2681 case Builtin::BI__builtin_parityll: {
2685 llvm::Type *ArgType = ArgValue->getType();
2689 Value *Tmp = Builder.CreateCall(F, ArgValue);
2690 Value *
Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
2691 if (
Result->getType() != ResultType)
2692 Result = Builder.CreateIntCast(
Result, ResultType,
true,
2696 case Builtin::BI__lzcnt16:
2697 case Builtin::BI__lzcnt:
2698 case Builtin::BI__lzcnt64: {
2701 llvm::Type *ArgType = ArgValue->getType();
2705 Value *
Result = Builder.CreateCall(F, {ArgValue, Builder.getFalse()});
2706 if (
Result->getType() != ResultType)
2707 Result = Builder.CreateIntCast(
Result, ResultType,
true,
2711 case Builtin::BI__popcnt16:
2712 case Builtin::BI__popcnt:
2713 case Builtin::BI__popcnt64:
2714 case Builtin::BI__builtin_popcount:
2715 case Builtin::BI__builtin_popcountl:
2716 case Builtin::BI__builtin_popcountll: {
2719 llvm::Type *ArgType = ArgValue->getType();
2724 if (
Result->getType() != ResultType)
2725 Result = Builder.CreateIntCast(
Result, ResultType,
true,
2729 case Builtin::BI__builtin_unpredictable: {
2735 case Builtin::BI__builtin_expect: {
2737 llvm::Type *ArgType = ArgValue->getType();
2748 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue},
"expval");
2751 case Builtin::BI__builtin_expect_with_probability: {
2753 llvm::Type *ArgType = ArgValue->getType();
2756 llvm::APFloat Probability(0.0);
2759 assert(EvalSucceed &&
"probability should be able to evaluate as float");
2761 bool LoseInfo =
false;
2762 Probability.convert(llvm::APFloat::IEEEdouble(),
2763 llvm::RoundingMode::Dynamic, &LoseInfo);
2765 Constant *Confidence = ConstantFP::get(Ty, Probability);
2772 Function *FnExpect =
2775 FnExpect, {ArgValue, ExpectedValue, Confidence},
"expval");
2778 case Builtin::BI__builtin_assume_aligned: {
2781 Value *OffsetValue =
2786 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
2787 AlignmentCI = ConstantInt::get(AlignmentCI->getType(),
2788 llvm::Value::MaximumAlignment);
2792 AlignmentCI, OffsetValue);
2795 case Builtin::BI__assume:
2796 case Builtin::BI__builtin_assume: {
2802 return RValue::get(Builder.CreateCall(FnAssume, ArgValue));
2804 case Builtin::BI__arithmetic_fence: {
2807 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
2808 llvm::FastMathFlags FMF = Builder.getFastMathFlags();
2809 bool isArithmeticFenceEnabled =
2810 FMF.allowReassoc() &&
2813 if (ArgType->isComplexType()) {
2814 if (isArithmeticFenceEnabled) {
2817 Value *Real = Builder.CreateArithmeticFence(ComplexVal.first,
2819 Value *Imag = Builder.CreateArithmeticFence(ComplexVal.second,
2824 Value *Real = ComplexVal.first;
2825 Value *Imag = ComplexVal.second;
2829 if (isArithmeticFenceEnabled)
2831 Builder.CreateArithmeticFence(ArgValue,
ConvertType(ArgType)));
2834 case Builtin::BI__builtin_bswap16:
2835 case Builtin::BI__builtin_bswap32:
2836 case Builtin::BI__builtin_bswap64:
2837 case Builtin::BI_byteswap_ushort:
2838 case Builtin::BI_byteswap_ulong:
2839 case Builtin::BI_byteswap_uint64: {
2842 case Builtin::BI__builtin_bitreverse8:
2843 case Builtin::BI__builtin_bitreverse16:
2844 case Builtin::BI__builtin_bitreverse32:
2845 case Builtin::BI__builtin_bitreverse64: {
2848 case Builtin::BI__builtin_rotateleft8:
2849 case Builtin::BI__builtin_rotateleft16:
2850 case Builtin::BI__builtin_rotateleft32:
2851 case Builtin::BI__builtin_rotateleft64:
2852 case Builtin::BI_rotl8:
2853 case Builtin::BI_rotl16:
2854 case Builtin::BI_rotl:
2855 case Builtin::BI_lrotl:
2856 case Builtin::BI_rotl64:
2859 case Builtin::BI__builtin_rotateright8:
2860 case Builtin::BI__builtin_rotateright16:
2861 case Builtin::BI__builtin_rotateright32:
2862 case Builtin::BI__builtin_rotateright64:
2863 case Builtin::BI_rotr8:
2864 case Builtin::BI_rotr16:
2865 case Builtin::BI_rotr:
2866 case Builtin::BI_lrotr:
2867 case Builtin::BI_rotr64:
2870 case Builtin::BI__builtin_constant_p: {
2877 if (!ArgType->isIntegralOrEnumerationType() && !ArgType->isFloatingType() &&
2878 !ArgType->isObjCObjectPointerType() && !ArgType->isBlockPointerType())
2881 return RValue::get(ConstantInt::get(ResultType, 0));
2886 return RValue::get(ConstantInt::get(ResultType, 0));
2889 if (ArgType->isObjCObjectPointerType()) {
2893 ArgValue = Builder.CreateBitCast(ArgValue,
ConvertType(ArgType));
2898 if (
Result->getType() != ResultType)
2899 Result = Builder.CreateIntCast(
Result, ResultType,
false);
2902 case Builtin::BI__builtin_dynamic_object_size:
2903 case Builtin::BI__builtin_object_size: {
2910 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
2912 nullptr, IsDynamic));
2914 case Builtin::BI__builtin_prefetch: {
2920 llvm::ConstantInt::get(
Int32Ty, 3);
2923 return RValue::get(Builder.CreateCall(F, {Address, RW, Locality, Data}));
2925 case Builtin::BI__builtin_readcyclecounter: {
2929 case Builtin::BI__builtin___clear_cache: {
2933 return RValue::get(Builder.CreateCall(F, {Begin, End}));
2935 case Builtin::BI__builtin_trap:
2937 case Builtin::BI__debugbreak:
2939 case Builtin::BI__builtin_unreachable: {
2948 case Builtin::BI__builtin_powi:
2949 case Builtin::BI__builtin_powif:
2950 case Builtin::BI__builtin_powil: {
2954 if (Builder.getIsFPConstrained()) {
2955 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
2956 Function *F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_powi,
2958 return RValue::get(Builder.CreateConstrainedFPCall(F, { Src0, Src1 }));
2962 { Src0->getType(), Src1->getType() });
2963 return RValue::get(Builder.CreateCall(F, { Src0, Src1 }));
2965 case Builtin::BI__builtin_isgreater:
2966 case Builtin::BI__builtin_isgreaterequal:
2967 case Builtin::BI__builtin_isless:
2968 case Builtin::BI__builtin_islessequal:
2969 case Builtin::BI__builtin_islessgreater:
2970 case Builtin::BI__builtin_isunordered: {
2973 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
2978 switch (BuiltinID) {
2979 default: llvm_unreachable(
"Unknown ordered comparison");
2980 case Builtin::BI__builtin_isgreater:
2981 LHS = Builder.CreateFCmpOGT(LHS, RHS,
"cmp");
2983 case Builtin::BI__builtin_isgreaterequal:
2984 LHS = Builder.CreateFCmpOGE(LHS, RHS,
"cmp");
2986 case Builtin::BI__builtin_isless:
2987 LHS = Builder.CreateFCmpOLT(LHS, RHS,
"cmp");
2989 case Builtin::BI__builtin_islessequal:
2990 LHS = Builder.CreateFCmpOLE(LHS, RHS,
"cmp");
2992 case Builtin::BI__builtin_islessgreater:
2993 LHS = Builder.CreateFCmpONE(LHS, RHS,
"cmp");
2995 case Builtin::BI__builtin_isunordered:
2996 LHS = Builder.CreateFCmpUNO(LHS, RHS,
"cmp");
3002 case Builtin::BI__builtin_isnan: {
3003 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3005 llvm::Type *Ty =
V->getType();
3006 const llvm::fltSemantics &Semantics = Ty->getFltSemantics();
3007 if (!Builder.getIsFPConstrained() ||
3008 Builder.getDefaultConstrainedExcept() == fp::ebIgnore ||
3010 V = Builder.CreateFCmpUNO(
V,
V,
"cmp");
3019 unsigned bitsize = Ty->getScalarSizeInBits();
3020 llvm::IntegerType *
IntTy = Builder.getIntNTy(bitsize);
3022 APInt AndMask = APInt::getSignedMaxValue(bitsize);
3024 Builder.CreateAnd(IntV, llvm::ConstantInt::get(
IntTy, AndMask));
3025 APInt ExpMask = APFloat::getInf(Semantics).bitcastToAPInt();
3027 Builder.CreateSub(llvm::ConstantInt::get(
IntTy, ExpMask), AbsV);
3029 V = Builder.CreateLShr(Sub, llvm::ConstantInt::get(
IntTy, bitsize - 1));
3035 case Builtin::BI__builtin_elementwise_abs: {
3040 QT = VecTy->getElementType();
3042 Result = Builder.CreateBinaryIntrinsic(
3044 Builder.getFalse(),
nullptr,
"elt.abs");
3051 case Builtin::BI__builtin_elementwise_ceil:
3054 case Builtin::BI__builtin_elementwise_floor:
3057 case Builtin::BI__builtin_elementwise_roundeven:
3060 case Builtin::BI__builtin_elementwise_trunc:
3064 case Builtin::BI__builtin_elementwise_add_sat:
3065 case Builtin::BI__builtin_elementwise_sub_sat: {
3069 assert(Op0->getType()->isIntOrIntVectorTy() &&
"integer type expected");
3072 Ty = VecTy->getElementType();
3075 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
3076 Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
3078 Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;
3079 Result = Builder.CreateBinaryIntrinsic(Opc, Op0, Op1,
nullptr,
"elt.sat");
3083 case Builtin::BI__builtin_elementwise_max: {
3087 if (Op0->getType()->isIntOrIntVectorTy()) {
3090 Ty = VecTy->getElementType();
3092 ? llvm::Intrinsic::smax
3093 : llvm::Intrinsic::umax,
3094 Op0, Op1,
nullptr,
"elt.max");
3096 Result = Builder.CreateMaxNum(Op0, Op1,
"elt.max");
3099 case Builtin::BI__builtin_elementwise_min: {
3103 if (Op0->getType()->isIntOrIntVectorTy()) {
3106 Ty = VecTy->getElementType();
3108 ? llvm::Intrinsic::smin
3109 : llvm::Intrinsic::umin,
3110 Op0, Op1,
nullptr,
"elt.min");
3112 Result = Builder.CreateMinNum(Op0, Op1,
"elt.min");
3116 case Builtin::BI__builtin_reduce_max: {
3117 auto GetIntrinsicID = [](
QualType QT) {
3119 QT = VecTy->getElementType();
3121 return llvm::Intrinsic::vector_reduce_smax;
3123 return llvm::Intrinsic::vector_reduce_umax;
3125 return llvm::Intrinsic::vector_reduce_fmax;
3128 *
this, E, GetIntrinsicID(E->
getArg(0)->
getType()),
"rdx.min"));
3131 case Builtin::BI__builtin_reduce_min: {
3132 auto GetIntrinsicID = [](
QualType QT) {
3134 QT = VecTy->getElementType();
3136 return llvm::Intrinsic::vector_reduce_smin;
3138 return llvm::Intrinsic::vector_reduce_umin;
3140 return llvm::Intrinsic::vector_reduce_fmin;
3144 *
this, E, GetIntrinsicID(E->
getArg(0)->
getType()),
"rdx.min"));
3147 case Builtin::BI__builtin_reduce_add:
3149 *
this, E, llvm::Intrinsic::vector_reduce_add,
"rdx.add"));
3150 case Builtin::BI__builtin_reduce_mul:
3152 *
this, E, llvm::Intrinsic::vector_reduce_mul,
"rdx.mul"));
3153 case Builtin::BI__builtin_reduce_xor:
3155 *
this, E, llvm::Intrinsic::vector_reduce_xor,
"rdx.xor"));
3156 case Builtin::BI__builtin_reduce_or:
3158 *
this, E, llvm::Intrinsic::vector_reduce_or,
"rdx.or"));
3159 case Builtin::BI__builtin_reduce_and:
3161 *
this, E, llvm::Intrinsic::vector_reduce_and,
"rdx.and"));
3163 case Builtin::BI__builtin_matrix_transpose: {
3166 MatrixBuilder MB(Builder);
3167 Value *
Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
3168 MatrixTy->getNumColumns());
3172 case Builtin::BI__builtin_matrix_column_major_load: {
3173 MatrixBuilder MB(Builder);
3178 assert(PtrTy &&
"arg0 must be of pointer type");
3179 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
3187 ResultTy->getNumRows(), ResultTy->getNumColumns(),
3192 case Builtin::BI__builtin_matrix_column_major_store: {
3193 MatrixBuilder MB(Builder);
3200 assert(PtrTy &&
"arg1 must be of pointer type");
3201 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
3207 Stride, IsVolatile, MatrixTy->getNumRows(), MatrixTy->getNumColumns());
3211 case Builtin::BIfinite:
3212 case Builtin::BI__finite:
3213 case Builtin::BIfinitef:
3214 case Builtin::BI__finitef:
3215 case Builtin::BIfinitel:
3216 case Builtin::BI__finitel:
3217 case Builtin::BI__builtin_isinf:
3218 case Builtin::BI__builtin_isfinite: {
3222 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3224 llvm::Type *Ty =
V->getType();
3225 if (!Builder.getIsFPConstrained() ||
3226 Builder.getDefaultConstrainedExcept() == fp::ebIgnore ||
3229 Constant *Infinity = ConstantFP::getInfinity(
V->getType());
3230 CmpInst::Predicate Pred = (BuiltinID == Builtin::BI__builtin_isinf)
3232 : CmpInst::FCMP_ONE;
3233 Value *FCmp = Builder.CreateFCmp(Pred, Fabs, Infinity,
"cmpinf");
3243 unsigned bitsize = Ty->getScalarSizeInBits();
3244 llvm::IntegerType *
IntTy = Builder.getIntNTy(bitsize);
3246 Value *Shl1 = Builder.CreateShl(IntV, 1);
3247 const llvm::fltSemantics &Semantics = Ty->getFltSemantics();
3248 APInt ExpMask = APFloat::getInf(Semantics).bitcastToAPInt();
3249 Value *ExpMaskShl1 = llvm::ConstantInt::get(
IntTy, ExpMask.shl(1));
3250 if (BuiltinID == Builtin::BI__builtin_isinf)
3251 V = Builder.CreateICmpEQ(Shl1, ExpMaskShl1);
3253 V = Builder.CreateICmpULT(Shl1, ExpMaskShl1);
3257 case Builtin::BI__builtin_isinf_sign: {
3259 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3263 Value *IsInf = Builder.CreateFCmpOEQ(
3264 AbsArg, ConstantFP::getInfinity(Arg->getType()),
"isinf");
3270 Value *NegativeOne = ConstantInt::get(
IntTy, -1);
3271 Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One);
3272 Value *
Result = Builder.CreateSelect(IsInf, SignResult, Zero);
3276 case Builtin::BI__builtin_isnormal: {
3278 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3281 Value *
Eq = Builder.CreateFCmpOEQ(
V,
V,
"iseq");
3284 Value *IsLessThanInf =
3285 Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(
V->getType()),
"isinf");
3286 APFloat Smallest = APFloat::getSmallestNormalized(
3289 Builder.CreateFCmpUGE(Abs, ConstantFP::get(
V->getContext(), Smallest),
3291 V = Builder.CreateAnd(
Eq, IsLessThanInf,
"and");
3292 V = Builder.CreateAnd(
V, IsNormal,
"and");
3296 case Builtin::BI__builtin_flt_rounds: {
3301 if (
Result->getType() != ResultType)
3302 Result = Builder.CreateIntCast(
Result, ResultType,
true,
3307 case Builtin::BI__builtin_fpclassify: {
3308 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3314 BasicBlock *
Begin = Builder.GetInsertBlock();
3316 Builder.SetInsertPoint(End);
3319 "fpclassify_result");
3322 Builder.SetInsertPoint(
Begin);
3323 Value *IsZero = Builder.CreateFCmpOEQ(
V, Constant::getNullValue(Ty),
3327 Builder.CreateCondBr(IsZero, End, NotZero);
3331 Builder.SetInsertPoint(NotZero);
3332 Value *IsNan = Builder.CreateFCmpUNO(
V,
V,
"cmp");
3335 Builder.CreateCondBr(IsNan, End, NotNan);
3336 Result->addIncoming(NanLiteral, NotZero);
3339 Builder.SetInsertPoint(NotNan);
3342 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(
V->getType()),
3346 Builder.CreateCondBr(IsInf, End, NotInf);
3347 Result->addIncoming(InfLiteral, NotNan);
3350 Builder.SetInsertPoint(NotInf);
3351 APFloat Smallest = APFloat::getSmallestNormalized(
3354 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(
V->getContext(), Smallest),
3356 Value *NormalResult =
3359 Builder.CreateBr(End);
3360 Result->addIncoming(NormalResult, NotInf);
3363 Builder.SetInsertPoint(End);
3367 case Builtin::BIalloca:
3368 case Builtin::BI_alloca:
3369 case Builtin::BI__builtin_alloca_uninitialized:
3370 case Builtin::BI__builtin_alloca: {
3374 const Align SuitableAlignmentInBytes =
3378 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
3379 AI->setAlignment(SuitableAlignmentInBytes);
3380 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
3385 case Builtin::BI__builtin_alloca_with_align_uninitialized:
3386 case Builtin::BI__builtin_alloca_with_align: {
3390 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
3391 const Align AlignmentInBytes =
3393 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
3394 AI->setAlignment(AlignmentInBytes);
3395 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
3400 case Builtin::BIbzero:
3401 case Builtin::BI__builtin_bzero: {
3406 Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal,
false);
3409 case Builtin::BImemcpy:
3410 case Builtin::BI__builtin_memcpy:
3411 case Builtin::BImempcpy:
3412 case Builtin::BI__builtin_mempcpy: {
3420 Builder.CreateMemCpy(Dest, Src, SizeVal,
false);
3421 if (BuiltinID == Builtin::BImempcpy ||
3422 BuiltinID == Builtin::BI__builtin_mempcpy)
3429 case Builtin::BI__builtin_memcpy_inline: {
3438 Builder.CreateMemCpyInline(Dest, Src, Size);
3442 case Builtin::BI__builtin_char_memchr:
3443 BuiltinID = Builtin::BI__builtin_memchr;
3446 case Builtin::BI__builtin___memcpy_chk: {
3453 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
3454 if (
Size.ugt(DstSize))
3458 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
3459 Builder.CreateMemCpy(Dest, Src, SizeVal,
false);
3463 case Builtin::BI__builtin_objc_memmove_collectable: {
3468 DestAddr, SrcAddr, SizeVal);
3472 case Builtin::BI__builtin___memmove_chk: {
3479 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
3480 if (
Size.ugt(DstSize))
3484 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
3485 Builder.CreateMemMove(Dest, Src, SizeVal,
false);
3489 case Builtin::BImemmove:
3490 case Builtin::BI__builtin_memmove: {
3498 Builder.CreateMemMove(Dest, Src, SizeVal,
false);
3501 case Builtin::BImemset:
3502 case Builtin::BI__builtin_memset: {
3505 Builder.getInt8Ty());
3509 Builder.CreateMemSet(Dest, ByteVal, SizeVal,
false);
3512 case Builtin::BI__builtin_memset_inline: {
3520 Builder.CreateMemSetInline(Dest, ByteVal, Size);
3523 case Builtin::BI__builtin___memset_chk: {
3530 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
3531 if (
Size.ugt(DstSize))
3535 Builder.getInt8Ty());
3536 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
3537 Builder.CreateMemSet(Dest, ByteVal, SizeVal,
false);
3540 case Builtin::BI__builtin_wmemchr: {
3543 if (!
getTarget().getTriple().isOSMSVCRT())
3551 BasicBlock *Entry = Builder.GetInsertBlock();
3555 Value *SizeEq0 = Builder.CreateICmpEQ(Size, ConstantInt::get(
SizeTy, 0));
3556 Builder.CreateCondBr(SizeEq0, Exit, CmpEq);
3559 PHINode *StrPhi = Builder.CreatePHI(Str->getType(), 2);
3560 StrPhi->addIncoming(Str, Entry);
3561 PHINode *SizePhi = Builder.CreatePHI(
SizeTy, 2);
3562 SizePhi->addIncoming(Size, Entry);
3565 Value *StrCh = Builder.CreateAlignedLoad(WCharTy, StrPhi, WCharAlign);
3566 Value *FoundChr = Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 0);
3567 Value *StrEqChr = Builder.CreateICmpEQ(StrCh, Chr);
3568 Builder.CreateCondBr(StrEqChr, Exit,
Next);
3571 Value *NextStr = Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 1);
3572 Value *NextSize = Builder.CreateSub(SizePhi, ConstantInt::get(
SizeTy, 1));
3573 Value *NextSizeEq0 =
3574 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
3575 Builder.CreateCondBr(NextSizeEq0, Exit, CmpEq);
3576 StrPhi->addIncoming(NextStr,
Next);
3577 SizePhi->addIncoming(NextSize,
Next);
3580 PHINode *
Ret = Builder.CreatePHI(Str->getType(), 3);
3581 Ret->addIncoming(llvm::Constant::getNullValue(Str->getType()), Entry);
3582 Ret->addIncoming(llvm::Constant::getNullValue(Str->getType()),
Next);
3583 Ret->addIncoming(FoundChr, CmpEq);
3586 case Builtin::BI__builtin_wmemcmp: {
3589 if (!
getTarget().getTriple().isOSMSVCRT())
3598 BasicBlock *Entry = Builder.GetInsertBlock();
3603 Value *SizeEq0 = Builder.CreateICmpEQ(Size, ConstantInt::get(
SizeTy, 0));
3604 Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
3607 PHINode *DstPhi = Builder.CreatePHI(Dst->getType(), 2);
3608 DstPhi->addIncoming(Dst, Entry);
3609 PHINode *SrcPhi = Builder.CreatePHI(Src->getType(), 2);
3610 SrcPhi->addIncoming(Src, Entry);
3611 PHINode *SizePhi = Builder.CreatePHI(
SizeTy, 2);
3612 SizePhi->addIncoming(Size, Entry);
3615 Value *DstCh = Builder.CreateAlignedLoad(WCharTy, DstPhi, WCharAlign);
3616 Value *SrcCh = Builder.CreateAlignedLoad(WCharTy, SrcPhi, WCharAlign);
3617 Value *DstGtSrc = Builder.CreateICmpUGT(DstCh, SrcCh);
3618 Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
3621 Value *DstLtSrc = Builder.CreateICmpULT(DstCh, SrcCh);
3622 Builder.CreateCondBr(DstLtSrc, Exit,
Next);
3625 Value *NextDst = Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
3626 Value *NextSrc = Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
3627 Value *NextSize = Builder.CreateSub(SizePhi, ConstantInt::get(
SizeTy, 1));
3628 Value *NextSizeEq0 =
3629 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
3630 Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
3631 DstPhi->addIncoming(NextDst,
Next);
3632 SrcPhi->addIncoming(NextSrc,
Next);
3633 SizePhi->addIncoming(NextSize,
Next);
3636 PHINode *
Ret = Builder.CreatePHI(
IntTy, 4);
3637 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Entry);
3638 Ret->addIncoming(ConstantInt::get(
IntTy, 1), CmpGT);
3639 Ret->addIncoming(ConstantInt::get(
IntTy, -1), CmpLT);
3643 case Builtin::BI__builtin_dwarf_cfa: {
3658 case Builtin::BI__builtin_return_address: {
3664 case Builtin::BI_ReturnAddress: {
3666 return RValue::get(Builder.CreateCall(F, Builder.getInt32(0)));
3668 case Builtin::BI__builtin_frame_address: {
3674 case Builtin::BI__builtin_extract_return_addr: {
3679 case Builtin::BI__builtin_frob_return_addr: {
3684 case Builtin::BI__builtin_dwarf_sp_column: {
3685 llvm::IntegerType *Ty
3692 return RValue::get(llvm::ConstantInt::get(Ty, Column,
true));
3694 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
3700 case Builtin::BI__builtin_eh_return: {
3705 assert((
IntTy->getBitWidth() == 32 ||
IntTy->getBitWidth() == 64) &&
3706 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
3709 : Intrinsic::eh_return_i64);
3710 Builder.CreateCall(F, {Int, Ptr});
3711 Builder.CreateUnreachable();
3718 case Builtin::BI__builtin_unwind_init: {
3722 case Builtin::BI__builtin_extend_pointer: {
3747 case Builtin::BI__builtin_setjmp: {
3752 Value *FrameAddr = Builder.CreateCall(
3754 ConstantInt::get(
Int32Ty, 0));
3755 Builder.CreateStore(FrameAddr, Buf);
3760 Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Buf, 2);
3761 Builder.CreateStore(StackAddr, StackSaveSlot);
3765 Buf = Builder.CreateElementBitCast(Buf,
Int8Ty);
3768 case Builtin::BI__builtin_longjmp: {
3770 Buf = Builder.CreateBitCast(Buf,
Int8PtrTy);
3773 Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
3776 Builder.CreateUnreachable();
3783 case Builtin::BI__builtin_launder: {
3788 Ptr = Builder.CreateLaunderInvariantGroup(Ptr);
3792 case Builtin::BI__sync_fetch_and_add:
3793 case Builtin::BI__sync_fetch_and_sub:
3794 case Builtin::BI__sync_fetch_and_or:
3795 case Builtin::BI__sync_fetch_and_and:
3796 case Builtin::BI__sync_fetch_and_xor:
3797 case Builtin::BI__sync_fetch_and_nand:
3798 case Builtin::BI__sync_add_and_fetch:
3799 case Builtin::BI__sync_sub_and_fetch:
3800 case Builtin::BI__sync_and_and_fetch:
3801 case Builtin::BI__sync_or_and_fetch:
3802 case Builtin::BI__sync_xor_and_fetch:
3803 case Builtin::BI__sync_nand_and_fetch:
3804 case Builtin::BI__sync_val_compare_and_swap:
3805 case Builtin::BI__sync_bool_compare_and_swap:
3806 case Builtin::BI__sync_lock_test_and_set:
3807 case Builtin::BI__sync_lock_release:
3808 case Builtin::BI__sync_swap:
3809 llvm_unreachable(
"Shouldn't make it through sema");
3810 case Builtin::BI__sync_fetch_and_add_1:
3811 case Builtin::BI__sync_fetch_and_add_2:
3812 case Builtin::BI__sync_fetch_and_add_4:
3813 case Builtin::BI__sync_fetch_and_add_8:
3814 case Builtin::BI__sync_fetch_and_add_16:
3816 case Builtin::BI__sync_fetch_and_sub_1:
3817 case Builtin::BI__sync_fetch_and_sub_2:
3818 case Builtin::BI__sync_fetch_and_sub_4:
3819 case Builtin::BI__sync_fetch_and_sub_8:
3820 case Builtin::BI__sync_fetch_and_sub_16:
3822 case Builtin::BI__sync_fetch_and_or_1:
3823 case Builtin::BI__sync_fetch_and_or_2:
3824 case Builtin::BI__sync_fetch_and_or_4:
3825 case Builtin::BI__sync_fetch_and_or_8:
3826 case Builtin::BI__sync_fetch_and_or_16:
3828 case Builtin::BI__sync_fetch_and_and_1:
3829 case Builtin::BI__sync_fetch_and_and_2:
3830 case Builtin::BI__sync_fetch_and_and_4:
3831 case Builtin::BI__sync_fetch_and_and_8:
3832 case Builtin::BI__sync_fetch_and_and_16:
3834 case Builtin::BI__sync_fetch_and_xor_1:
3835 case Builtin::BI__sync_fetch_and_xor_2:
3836 case Builtin::BI__sync_fetch_and_xor_4:
3837 case Builtin::BI__sync_fetch_and_xor_8:
3838 case Builtin::BI__sync_fetch_and_xor_16:
3840 case Builtin::BI__sync_fetch_and_nand_1:
3841 case Builtin::BI__sync_fetch_and_nand_2:
3842 case Builtin::BI__sync_fetch_and_nand_4:
3843 case Builtin::BI__sync_fetch_and_nand_8:
3844 case Builtin::BI__sync_fetch_and_nand_16:
3848 case Builtin::BI__sync_fetch_and_min:
3850 case Builtin::BI__sync_fetch_and_max:
3852 case Builtin::BI__sync_fetch_and_umin:
3854 case Builtin::BI__sync_fetch_and_umax:
3857 case Builtin::BI__sync_add_and_fetch_1:
3858 case Builtin::BI__sync_add_and_fetch_2:
3859 case Builtin::BI__sync_add_and_fetch_4:
3860 case Builtin::BI__sync_add_and_fetch_8:
3861 case Builtin::BI__sync_add_and_fetch_16:
3863 llvm::Instruction::Add);
3864 case Builtin::BI__sync_sub_and_fetch_1:
3865 case Builtin::BI__sync_sub_and_fetch_2:
3866 case Builtin::BI__sync_sub_and_fetch_4:
3867 case Builtin::BI__sync_sub_and_fetch_8:
3868 case Builtin::BI__sync_sub_and_fetch_16:
3870 llvm::Instruction::Sub);
3871 case Builtin::BI__sync_and_and_fetch_1:
3872 case Builtin::BI__sync_and_and_fetch_2:
3873 case Builtin::BI__sync_and_and_fetch_4:
3874 case Builtin::BI__sync_and_and_fetch_8:
3875 case Builtin::BI__sync_and_and_fetch_16:
3877 llvm::Instruction::And);
3878 case Builtin::BI__sync_or_and_fetch_1:
3879 case Builtin::BI__sync_or_and_fetch_2:
3880 case Builtin::BI__sync_or_and_fetch_4:
3881 case Builtin::BI__sync_or_and_fetch_8:
3882 case Builtin::BI__sync_or_and_fetch_16:
3884 llvm::Instruction::Or);
3885 case Builtin::BI__sync_xor_and_fetch_1:
3886 case Builtin::BI__sync_xor_and_fetch_2:
3887 case Builtin::BI__sync_xor_and_fetch_4:
3888 case Builtin::BI__sync_xor_and_fetch_8:
3889 case Builtin::BI__sync_xor_and_fetch_16:
3891 llvm::Instruction::Xor);
3892 case Builtin::BI__sync_nand_and_fetch_1:
3893 case Builtin::BI__sync_nand_and_fetch_2:
3894 case Builtin::BI__sync_nand_and_fetch_4:
3895 case Builtin::BI__sync_nand_and_fetch_8:
3896 case Builtin::BI__sync_nand_and_fetch_16:
3898 llvm::Instruction::And,
true);
3900 case Builtin::BI__sync_val_compare_and_swap_1:
3901 case Builtin::BI__sync_val_compare_and_swap_2:
3902 case Builtin::BI__sync_val_compare_and_swap_4:
3903 case Builtin::BI__sync_val_compare_and_swap_8:
3904 case Builtin::BI__sync_val_compare_and_swap_16:
3907 case Builtin::BI__sync_bool_compare_and_swap_1:
3908 case Builtin::BI__sync_bool_compare_and_swap_2:
3909 case Builtin::BI__sync_bool_compare_and_swap_4:
3910 case Builtin::BI__sync_bool_compare_and_swap_8:
3911 case Builtin::BI__sync_bool_compare_and_swap_16:
3914 case Builtin::BI__sync_swap_1:
3915 case Builtin::BI__sync_swap_2:
3916 case Builtin::BI__sync_swap_4:
3917 case Builtin::BI__sync_swap_8:
3918 case Builtin::BI__sync_swap_16:
3921 case Builtin::BI__sync_lock_test_and_set_1:
3922 case Builtin::BI__sync_lock_test_and_set_2:
3923 case Builtin::BI__sync_lock_test_and_set_4:
3924 case Builtin::BI__sync_lock_test_and_set_8:
3925 case Builtin::BI__sync_lock_test_and_set_16:
3928 case Builtin::BI__sync_lock_release_1:
3929 case Builtin::BI__sync_lock_release_2:
3930 case Builtin::BI__sync_lock_release_4:
3931 case Builtin::BI__sync_lock_release_8:
3932 case Builtin::BI__sync_lock_release_16: {
3938 Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
3939 llvm::StoreInst *
Store =
3940 Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr,
3942 Store->setAtomic(llvm::AtomicOrdering::Release);
3946 case Builtin::BI__sync_synchronize: {
3954 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
3958 case Builtin::BI__builtin_nontemporal_load:
3960 case Builtin::BI__builtin_nontemporal_store:
3962 case Builtin::BI__c11_atomic_is_lock_free:
3963 case Builtin::BI__atomic_is_lock_free: {
3967 const char *LibCallName =
"__atomic_is_lock_free";
3971 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
3985 case Builtin::BI__atomic_test_and_set: {
3993 unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
3994 Ptr = Builder.CreateBitCast(Ptr,
Int8Ty->getPointerTo(AddrSpace));
3995 Value *NewVal = Builder.getInt8(1);
3997 if (isa<llvm::ConstantInt>(Order)) {
3999 AtomicRMWInst *
Result =
nullptr;
4003 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4004 llvm::AtomicOrdering::Monotonic);
4008 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4009 llvm::AtomicOrdering::Acquire);
4012 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4013 llvm::AtomicOrdering::Release);
4017 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4018 llvm::AtomicOrdering::AcquireRelease);
4021 Result = Builder.CreateAtomicRMW(
4022 llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4023 llvm::AtomicOrdering::SequentiallyConsistent);
4026 Result->setVolatile(Volatile);
4032 llvm::BasicBlock *BBs[5] = {
4039 llvm::AtomicOrdering Orders[5] = {
4040 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
4041 llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
4042 llvm::AtomicOrdering::SequentiallyConsistent};
4044 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(),
false);
4045 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
4047 Builder.SetInsertPoint(ContBB);
4048 PHINode *
Result = Builder.CreatePHI(
Int8Ty, 5,
"was_set");
4050 for (
unsigned i = 0; i < 5; ++i) {
4051 Builder.SetInsertPoint(BBs[i]);
4052 AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
4053 Ptr, NewVal, Orders[i]);
4054 RMW->setVolatile(Volatile);
4055 Result->addIncoming(RMW, BBs[i]);
4056 Builder.CreateBr(ContBB);
4059 SI->addCase(Builder.getInt32(0), BBs[0]);
4060 SI->addCase(Builder.getInt32(1), BBs[1]);
4061 SI->addCase(Builder.getInt32(2), BBs[1]);
4062 SI->addCase(Builder.getInt32(3), BBs[2]);
4063 SI->addCase(Builder.getInt32(4), BBs[3]);
4064 SI->addCase(Builder.getInt32(5), BBs[4]);
4066 Builder.SetInsertPoint(ContBB);
4070 case Builtin::BI__atomic_clear: {
4073 PtrTy->castAs<
PointerType>()->getPointeeType().isVolatileQualified();
4076 Ptr = Builder.CreateElementBitCast(Ptr,
Int8Ty);
4077 Value *NewVal = Builder.getInt8(0);
4079 if (isa<llvm::ConstantInt>(Order)) {
4081 StoreInst *
Store = Builder.CreateStore(NewVal, Ptr, Volatile);
4085 Store->setOrdering(llvm::AtomicOrdering::Monotonic);
4088 Store->setOrdering(llvm::AtomicOrdering::Release);
4091 Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
4099 llvm::BasicBlock *BBs[3] = {
4104 llvm::AtomicOrdering Orders[3] = {
4105 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release,
4106 llvm::AtomicOrdering::SequentiallyConsistent};
4108 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(),
false);
4109 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
4111 for (
unsigned i = 0; i < 3; ++i) {
4112 Builder.SetInsertPoint(BBs[i]);
4113 StoreInst *
Store = Builder.CreateStore(NewVal, Ptr, Volatile);
4114 Store->setOrdering(Orders[i]);
4115 Builder.CreateBr(ContBB);
4118 SI->addCase(Builder.getInt32(0), BBs[0]);
4119 SI->addCase(Builder.getInt32(3), BBs[1]);
4120 SI->addCase(Builder.getInt32(5), BBs[2]);
4122 Builder.SetInsertPoint(ContBB);
4126 case Builtin::BI__atomic_thread_fence:
4127 case Builtin::BI__atomic_signal_fence:
4128 case Builtin::BI__c11_atomic_thread_fence:
4129 case Builtin::BI__c11_atomic_signal_fence: {
4130 llvm::SyncScope::ID SSID;
4131 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
4132 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
4133 SSID = llvm::SyncScope::SingleThread;
4135 SSID = llvm::SyncScope::System;
4137 if (isa<llvm::ConstantInt>(Order)) {
4145 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
4148 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
4151 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
4154 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
4160 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
4167 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(),
false);
4168 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB);
4170 Builder.SetInsertPoint(AcquireBB);
4171 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
4172 Builder.CreateBr(ContBB);
4173 SI->addCase(Builder.getInt32(1), AcquireBB);
4174 SI->addCase(Builder.getInt32(2), AcquireBB);
4176 Builder.SetInsertPoint(ReleaseBB);
4177 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
4178 Builder.CreateBr(ContBB);
4179 SI->addCase(Builder.getInt32(3), ReleaseBB);
4181 Builder.SetInsertPoint(AcqRelBB);
4182 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
4183 Builder.CreateBr(ContBB);
4184 SI->addCase(Builder.getInt32(4), AcqRelBB);
4186 Builder.SetInsertPoint(SeqCstBB);
4187 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
4188 Builder.CreateBr(ContBB);
4189 SI->addCase(Builder.getInt32(5), SeqCstBB);
4191 Builder.SetInsertPoint(ContBB);
4195 case Builtin::BI__builtin_signbit:
4196 case Builtin::BI__builtin_signbitf:
4197 case Builtin::BI__builtin_signbitl: {
4202 case Builtin::BI__warn_memset_zero_len:
4204 case Builtin::BI__annotation: {
4209 assert(Str->getCharByteWidth() == 2);
4210 StringRef WideBytes = Str->getBytes();
4211 std::string StrUtf8;
4212 if (!convertUTF16ToUTF8String(
4213 makeArrayRef(WideBytes.data(), WideBytes.size()), StrUtf8)) {
4217 Strings.push_back(llvm::MDString::get(
getLLVMContext(), StrUtf8));
4224 Builder.CreateCall(F, MetadataAsValue::get(
getLLVMContext(), StrTuple));
4227 case Builtin::BI__builtin_annotation: {
4239 case Builtin::BI__builtin_addcb:
4240 case Builtin::BI__builtin_addcs:
4241 case Builtin::BI__builtin_addc:
4242 case Builtin::BI__builtin_addcl:
4243 case Builtin::BI__builtin_addcll:
4244 case Builtin::BI__builtin_subcb:
4245 case Builtin::BI__builtin_subcs:
4246 case Builtin::BI__builtin_subc:
4247 case Builtin::BI__builtin_subcl:
4248 case Builtin::BI__builtin_subcll: {
4274 llvm::Intrinsic::ID IntrinsicId;
4275 switch (BuiltinID) {
4276 default: llvm_unreachable(
"Unknown multiprecision builtin id.");
4277 case Builtin::BI__builtin_addcb:
4278 case Builtin::BI__builtin_addcs:
4279 case Builtin::BI__builtin_addc:
4280 case Builtin::BI__builtin_addcl:
4281 case Builtin::BI__builtin_addcll:
4282 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
4284 case Builtin::BI__builtin_subcb:
4285 case Builtin::BI__builtin_subcs:
4286 case Builtin::BI__builtin_subc:
4287 case Builtin::BI__builtin_subcl:
4288 case Builtin::BI__builtin_subcll:
4289 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
4294 llvm::Value *Carry1;
4297 llvm::Value *Carry2;
4299 Sum1, Carryin, Carry2);
4300 llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2),
4302 Builder.CreateStore(CarryOut, CarryOutPtr);
4306 case Builtin::BI__builtin_add_overflow:
4307 case Builtin::BI__builtin_sub_overflow:
4308 case Builtin::BI__builtin_mul_overflow: {
4316 WidthAndSignedness LeftInfo =
4318 WidthAndSignedness RightInfo =
4320 WidthAndSignedness ResultInfo =
4327 RightInfo, ResultArg, ResultQTy,
4333 *
this, LeftArg, LeftInfo, RightArg, RightInfo, ResultArg, ResultQTy,
4336 WidthAndSignedness EncompassingInfo =
4339 llvm::Type *EncompassingLLVMTy =
4344 llvm::Intrinsic::ID IntrinsicId;
4345 switch (BuiltinID) {
4347 llvm_unreachable(
"Unknown overflow builtin id.");
4348 case Builtin::BI__builtin_add_overflow:
4349 IntrinsicId = EncompassingInfo.Signed
4350 ? llvm::Intrinsic::sadd_with_overflow
4351 : llvm::Intrinsic::uadd_with_overflow;
4353 case Builtin::BI__builtin_sub_overflow:
4354 IntrinsicId = EncompassingInfo.Signed
4355 ? llvm::Intrinsic::ssub_with_overflow
4356 : llvm::Intrinsic::usub_with_overflow;
4358 case Builtin::BI__builtin_mul_overflow:
4359 IntrinsicId = EncompassingInfo.Signed
4360 ? llvm::Intrinsic::smul_with_overflow
4361 : llvm::Intrinsic::umul_with_overflow;
4370 Left = Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
4371 Right = Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
4374 llvm::Value *Overflow, *
Result;
4377 if (EncompassingInfo.Width > ResultInfo.Width) {
4380 llvm::Value *ResultTrunc = Builder.CreateTrunc(
Result, ResultLLVMTy);
4384 llvm::Value *ResultTruncExt = Builder.CreateIntCast(
4385 ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
4386 llvm::Value *TruncationOverflow =
4387 Builder.CreateICmpNE(
Result, ResultTruncExt);
4389 Overflow = Builder.CreateOr(Overflow, TruncationOverflow);
4401 case Builtin::BI__builtin_uadd_overflow:
4402 case Builtin::BI__builtin_uaddl_overflow:
4403 case Builtin::BI__builtin_uaddll_overflow:
4404 case Builtin::BI__builtin_usub_overflow:
4405 case Builtin::BI__builtin_usubl_overflow:
4406 case Builtin::BI__builtin_usubll_overflow:
4407 case Builtin::BI__builtin_umul_overflow:
4408 case Builtin::BI__builtin_umull_overflow:
4409 case Builtin::BI__builtin_umulll_overflow:
4410 case Builtin::BI__builtin_sadd_overflow:
4411 case Builtin::BI__builtin_saddl_overflow:
4412 case Builtin::BI__builtin_saddll_overflow:
4413 case Builtin::BI__builtin_ssub_overflow:
4414 case Builtin::BI__builtin_ssubl_overflow:
4415 case Builtin::BI__builtin_ssubll_overflow:
4416 case Builtin::BI__builtin_smul_overflow:
4417 case Builtin::BI__builtin_smull_overflow:
4418 case Builtin::BI__builtin_smulll_overflow: {
4428 llvm::Intrinsic::ID IntrinsicId;
4429 switch (BuiltinID) {
4430 default: llvm_unreachable(
"Unknown overflow builtin id.");
4431 case Builtin::BI__builtin_uadd_overflow:
4432 case Builtin::BI__builtin_uaddl_overflow:
4433 case Builtin::BI__builtin_uaddll_overflow:
4434 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
4436 case Builtin::BI__builtin_usub_overflow:
4437 case Builtin::BI__builtin_usubl_overflow:
4438 case Builtin::BI__builtin_usubll_overflow:
4439 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
4441 case Builtin::BI__builtin_umul_overflow:
4442 case Builtin::BI__builtin_umull_overflow:
4443 case Builtin::BI__builtin_umulll_overflow:
4444 IntrinsicId = llvm::Intrinsic::umul_with_overflow;
4446 case Builtin::BI__builtin_sadd_overflow:
4447 case Builtin::BI__builtin_saddl_overflow:
4448 case Builtin::BI__builtin_saddll_overflow:
4449 IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
4451 case Builtin::BI__builtin_ssub_overflow:
4452 case Builtin::BI__builtin_ssubl_overflow:
4453 case Builtin::BI__builtin_ssubll_overflow:
4454 IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
4456 case Builtin::BI__builtin_smul_overflow:
4457 case Builtin::BI__builtin_smull_overflow:
4458 case Builtin::BI__builtin_smulll_overflow:
4459 IntrinsicId = llvm::Intrinsic::smul_with_overflow;
4466 Builder.CreateStore(Sum, SumOutPtr);
4470 case Builtin::BIaddressof:
4471 case Builtin::BI__addressof:
4472 case Builtin::BI__builtin_addressof:
4474 case Builtin::BI__builtin_function_start:
4477 case Builtin::BI__builtin_operator_new:
4480 case Builtin::BI__builtin_operator_delete:
4484 case Builtin::BI__builtin_is_aligned:
4486 case Builtin::BI__builtin_align_up:
4488 case Builtin::BI__builtin_align_down:
4491 case Builtin::BI__noop:
4494 case Builtin::BI__builtin_call_with_static_chain: {
4501 case Builtin::BI_InterlockedExchange8:
4502 case Builtin::BI_InterlockedExchange16:
4503 case Builtin::BI_InterlockedExchange:
4504 case Builtin::BI_InterlockedExchangePointer:
4507 case Builtin::BI_InterlockedCompareExchangePointer:
4508 case Builtin::BI_InterlockedCompareExchangePointer_nf: {
4510 llvm::IntegerType *IntType =
4513 llvm::Type *IntPtrType = IntType->getPointerTo();
4515 llvm::Value *Destination =
4519 RTy = Exchange->getType();
4520 Exchange = Builder.CreatePtrToInt(Exchange, IntType);
4522 llvm::Value *Comparand =
4526 BuiltinID == Builtin::BI_InterlockedCompareExchangePointer_nf ?
4527 AtomicOrdering::Monotonic : AtomicOrdering::SequentiallyConsistent;
4529 auto Result = Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,
4530 Ordering, Ordering);
4531 Result->setVolatile(
true);
4537 case Builtin::BI_InterlockedCompareExchange8:
4538 case Builtin::BI_InterlockedCompareExchange16:
4539 case Builtin::BI_InterlockedCompareExchange:
4540 case Builtin::BI_InterlockedCompareExchange64:
4542 case Builtin::BI_InterlockedIncrement16:
4543 case Builtin::BI_InterlockedIncrement:
4546 case Builtin::BI_InterlockedDecrement16:
4547 case Builtin::BI_InterlockedDecrement:
4550 case Builtin::BI_InterlockedAnd8:
4551 case Builtin::BI_InterlockedAnd16:
4552 case Builtin::BI_InterlockedAnd:
4554 case Builtin::BI_InterlockedExchangeAdd8:
4555 case Builtin::BI_InterlockedExchangeAdd16:
4556 case Builtin::BI_InterlockedExchangeAdd:
4559 case Builtin::BI_InterlockedExchangeSub8:
4560 case Builtin::BI_InterlockedExchangeSub16:
4561 case Builtin::BI_InterlockedExchangeSub:
4564 case Builtin::BI_InterlockedOr8:
4565 case Builtin::BI_InterlockedOr16:
4566 case Builtin::BI_InterlockedOr:
4568 case Builtin::BI_InterlockedXor8:
4569 case Builtin::BI_InterlockedXor16:
4570 case Builtin::BI_InterlockedXor:
4573 case Builtin::BI_bittest64:
4574 case Builtin::BI_bittest:
4575 case Builtin::BI_bittestandcomplement64:
4576 case Builtin::BI_bittestandcomplement:
4577 case Builtin::BI_bittestandreset64:
4578 case Builtin::BI_bittestandreset:
4579 case Builtin::BI_bittestandset64:
4580 case Builtin::BI_bittestandset:
4581 case Builtin::BI_interlockedbittestandreset:
4582 case Builtin::BI_interlockedbittestandreset64:
4583 case Builtin::BI_interlockedbittestandset64:
4584 case Builtin::BI_interlockedbittestandset:
4585 case Builtin::BI_interlockedbittestandset_acq:
4586 case Builtin::BI_interlockedbittestandset_rel:
4587 case Builtin::BI_interlockedbittestandset_nf:
4588 case Builtin::BI_interlockedbittestandreset_acq:
4589 case Builtin::BI_interlockedbittestandreset_rel:
4590 case Builtin::BI_interlockedbittestandreset_nf:
4595 case Builtin::BI__iso_volatile_load8:
4596 case Builtin::BI__iso_volatile_load16:
4597 case Builtin::BI__iso_volatile_load32:
4598 case Builtin::BI__iso_volatile_load64:
4600 case Builtin::BI__iso_volatile_store8:
4601 case Builtin::BI__iso_volatile_store16:
4602 case Builtin::BI__iso_volatile_store32:
4603 case Builtin::BI__iso_volatile_store64:
4606 case Builtin::BI__exception_code:
4607 case Builtin::BI_exception_code:
4609 case Builtin::BI__exception_info:
4610 case Builtin::BI_exception_info:
4612 case Builtin::BI__abnormal_termination:
4613 case Builtin::BI_abnormal_termination:
4615 case Builtin::BI_setjmpex:
4620 case Builtin::BI_setjmp:
4623 if (
getTarget().getTriple().getArch() == llvm::Triple::x86)
4625 else if (
getTarget().getTriple().getArch() == llvm::Triple::aarch64)
4632 case Builtin::BImove:
4633 case Builtin::BImove_if_noexcept:
4634 case Builtin::BIforward:
4635 case Builtin::BIas_const:
4637 case Builtin::BI__GetExceptionInfo: {
4638 if (llvm::GlobalVariable *GV =
4644 case Builtin::BI__fastfail:
4647 case Builtin::BI__builtin_coro_size: {
4655 case Builtin::BI__builtin_coro_id:
4657 case Builtin::BI__builtin_coro_promise:
4659 case Builtin::BI__builtin_coro_resume:
4661 case Builtin::BI__builtin_coro_frame:
4663 case Builtin::BI__builtin_coro_noop:
4665 case Builtin::BI__builtin_coro_free:
4667 case Builtin::BI__builtin_coro_destroy:
4669 case Builtin::BI__builtin_coro_done:
4671 case Builtin::BI__builtin_coro_alloc:
4673 case Builtin::BI__builtin_coro_begin:
4675 case Builtin::BI__builtin_coro_end:
4677 case Builtin::BI__builtin_coro_suspend:
4681 case Builtin::BIread_pipe:
4682 case Builtin::BIwrite_pipe: {
4686 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
4687 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
4690 unsigned GenericAS =
4692 llvm::Type *I8PTy = llvm::PointerType::get(
4697 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_2"
4701 llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy,
Int32Ty,
Int32Ty};
4702 llvm::FunctionType *FTy = llvm::FunctionType::get(
4704 Value *BCast = Builder.CreatePointerCast(Arg1, I8PTy);
4707 {Arg0, BCast, PacketSize, PacketAlign}));
4710 "Illegal number of parameters to pipe function");
4711 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_4"
4714 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(),
Int32Ty, I8PTy,
4718 llvm::FunctionType *FTy = llvm::FunctionType::get(
4720 Value *BCast = Builder.CreatePointerCast(Arg3, I8PTy);
4723 if (Arg2->getType() !=
Int32Ty)
4724 Arg2 = Builder.CreateZExtOrTrunc(Arg2,
Int32Ty);
4727 {Arg0, Arg1, Arg2, BCast, PacketSize, PacketAlign}));
4732 case Builtin::BIreserve_read_pipe:
4733 case Builtin::BIreserve_write_pipe:
4734 case Builtin::BIwork_group_reserve_read_pipe:
4735 case Builtin::BIwork_group_reserve_write_pipe:
4736 case Builtin::BIsub_group_reserve_read_pipe:
4737 case Builtin::BIsub_group_reserve_write_pipe: {
4740 if (BuiltinID == Builtin::BIreserve_read_pipe)
4741 Name =
"__reserve_read_pipe";
4742 else if (BuiltinID == Builtin::BIreserve_write_pipe)
4743 Name =
"__reserve_write_pipe";
4744 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
4745 Name =
"__work_group_reserve_read_pipe";
4746 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
4747 Name =
"__work_group_reserve_write_pipe";
4748 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
4749 Name =
"__sub_group_reserve_read_pipe";
4751 Name =
"__sub_group_reserve_write_pipe";
4757 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
4758 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
4762 llvm::FunctionType *FTy = llvm::FunctionType::get(
4766 if (Arg1->getType() !=
Int32Ty)
4767 Arg1 = Builder.CreateZExtOrTrunc(Arg1,
Int32Ty);
4769 {Arg0, Arg1, PacketSize, PacketAlign}));
4773 case Builtin::BIcommit_read_pipe:
4774 case Builtin::BIcommit_write_pipe:
4775 case Builtin::BIwork_group_commit_read_pipe:
4776 case Builtin::BIwork_group_commit_write_pipe:
4777 case Builtin::BIsub_group_commit_read_pipe:
4778 case Builtin::BIsub_group_commit_write_pipe: {
4780 if (BuiltinID == Builtin::BIcommit_read_pipe)
4781 Name =
"__commit_read_pipe";
4782 else if (BuiltinID == Builtin::BIcommit_write_pipe)
4783 Name =
"__commit_write_pipe";
4784 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
4785 Name =
"__work_group_commit_read_pipe";
4786 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
4787 Name =
"__work_group_commit_write_pipe";
4788 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
4789 Name =
"__sub_group_commit_read_pipe";
4791 Name =
"__sub_group_commit_write_pipe";
4796 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
4797 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
4800 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(),
Int32Ty,
Int32Ty};
4801 llvm::FunctionType *FTy =
4806 {Arg0, Arg1, PacketSize, PacketAlign}));
4809 case Builtin::BIget_pipe_num_packets:
4810 case Builtin::BIget_pipe_max_packets: {
4811 const char *BaseName;
4813 if (BuiltinID == Builtin::BIget_pipe_num_packets)
4814 BaseName =
"__get_pipe_num_packets";
4816 BaseName =
"__get_pipe_max_packets";
4817 std::string Name = std::string(BaseName) +
4818 std::string(PipeTy->isReadOnly() ?
"_ro" :
"_wo");
4823 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
4824 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
4826 llvm::FunctionType *FTy = llvm::FunctionType::get(
4830 {Arg0, PacketSize, PacketAlign}));
4834 case Builtin::BIto_global:
4835 case Builtin::BIto_local:
4836 case Builtin::BIto_private: {
4838 auto NewArgT = llvm::PointerType::get(
Int8Ty,
4840 auto NewRetT = llvm::PointerType::get(
Int8Ty,
4843 auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT},
false);
4844 llvm::Value *NewArg;
4845 if (Arg0->getType()->getPointerAddressSpace() !=
4846 NewArgT->getPointerAddressSpace())
4847 NewArg = Builder.CreateAddrSpaceCast(Arg0, NewArgT);
4849 NewArg = Builder.CreateBitOrPointerCast(Arg0, NewArgT);
4853 return RValue::get(Builder.CreateBitOrPointerCast(NewCall,
4859 case Builtin::BIenqueue_kernel: {
4864 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
4870 llvm::Value *
Range = NDRangeL.getAddress(*this).getPointer();
4871 llvm::Type *RangeTy = NDRangeL.getAddress(*this).getType();
4876 Name =
"__enqueue_kernel_basic";
4877 llvm::Type *ArgTys[] = {QueueTy,
Int32Ty, RangeTy, GenericVoidPtrTy,
4879 llvm::FunctionType *FTy = llvm::FunctionType::get(
4885 Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
4886 llvm::Value *
Block =
4887 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
4889 AttrBuilder B(Builder.getContext());
4890 B.addByValAttr(NDRangeL.getAddress(*this).getElementType());
4891 llvm::AttributeList ByValAttrSet =
4892 llvm::AttributeList::get(
CGM.
getModule().getContext(), 3U, B);
4896 {Queue, Flags, Range, Kernel, Block});
4897 RTCall->setAttributes(ByValAttrSet);
4900 assert(NumArgs >= 5 &&
"Invalid enqueue_kernel signature");
4904 auto CreateArrayForSizeVar = [=](
unsigned First)
4905 -> std::tuple<llvm::Value *, llvm::Value *, llvm::Value *> {
4906 llvm::APInt ArraySize(32, NumArgs -
First);
4911 llvm::Value *TmpPtr = Tmp.getPointer();
4914 llvm::Value *ElemPtr;
4917 auto *
Zero = llvm::ConstantInt::get(
IntTy, 0);
4918 for (
unsigned I =
First; I < NumArgs; ++I) {
4919 auto *Index = llvm::ConstantInt::get(
IntTy, I -
First);
4920 auto *GEP = Builder.CreateGEP(Tmp.getElementType(), TmpPtr,
4926 Builder.CreateAlignedStore(
4929 return std::tie(ElemPtr, TmpSize, TmpPtr);
4935 Name =
"__enqueue_kernel_varargs";
4939 Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
4940 auto *
Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
4941 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
4942 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(4);
4946 llvm::Value *
const Args[] = {Queue, Flags,
4950 llvm::Type *
const ArgTys[] = {
4951 QueueTy,
IntTy, RangeTy, GenericVoidPtrTy,
4952 GenericVoidPtrTy,
IntTy, ElemPtr->getType()};
4954 llvm::FunctionType *FTy = llvm::FunctionType::get(
Int32Ty, ArgTys,
false);
4964 llvm::PointerType *EventPtrTy = EventTy->getPointerTo(
4967 llvm::Value *NumEvents =
4973 llvm::Value *EventWaitList =
nullptr;
4976 EventWaitList = llvm::ConstantPointerNull::get(EventPtrTy);
4982 EventWaitList = Builder.CreatePointerCast(EventWaitList, EventPtrTy);
4984 llvm::Value *EventRet =
nullptr;
4987 EventRet = llvm::ConstantPointerNull::get(EventPtrTy);
4996 Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
4997 llvm::Value *
Block =
4998 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5000 std::vector<llvm::Type *> ArgTys = {
5002 EventPtrTy, EventPtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
5004 std::vector<llvm::Value *> Args = {Queue, Flags,
Range,
5005 NumEvents, EventWaitList, EventRet,
5010 Name =
"__enqueue_kernel_basic_events";
5011 llvm::FunctionType *FTy = llvm::FunctionType::get(
5019 Args.push_back(ConstantInt::get(
Int32Ty, NumArgs - 7));
5021 Name =
"__enqueue_kernel_events_varargs";
5023 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
5024 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(7);
5025 Args.push_back(ElemPtr);
5026 ArgTys.push_back(ElemPtr->getType());
5028 llvm::FunctionType *FTy = llvm::FunctionType::get(
5041 case Builtin::BIget_kernel_work_group_size: {
5042 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
5046 Value *
Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
5047 Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5050 llvm::FunctionType::get(
IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
5052 "__get_kernel_work_group_size_impl"),
5055 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
5056 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
5060 Value *
Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
5061 Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5064 llvm::FunctionType::get(
IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
5066 "__get_kernel_preferred_work_group_size_multiple_impl"),
5069 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
5070 case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
5071 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
5074 llvm::Value *NDRange = NDRangeL.getAddress(*this).getPointer();
5077 Value *
Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
5078 Value *
Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5080 BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
5081 ?
"__get_kernel_max_sub_group_size_for_ndrange_impl"
5082 :
"__get_kernel_sub_group_count_for_ndrange_impl";
5085 llvm::FunctionType::get(
5086 IntTy, {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
5089 {NDRange, Kernel, Block}));
5092 case Builtin::BI__builtin_store_half:
5093 case Builtin::BI__builtin_store_halff: {
5096 Value *HalfVal = Builder.CreateFPTrunc(Val, Builder.getHalfTy());
5099 case Builtin::BI__builtin_load_half: {
5102 return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getDoubleTy()));
5104 case Builtin::BI__builtin_load_halff: {
5107 return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getFloatTy()));
5109 case Builtin::BIprintf:
5110 if (
getTarget().getTriple().isNVPTX() ||
5121 case Builtin::BI__builtin_canonicalize:
5122 case Builtin::BI__builtin_canonicalizef:
5123 case Builtin::BI__builtin_canonicalizef16:
5124 case Builtin::BI__builtin_canonicalizel:
5127 case Builtin::BI__builtin_thread_pointer: {
5128 if (!
getContext().getTargetInfo().isTLSSupported())
5133 case Builtin::BI__builtin_os_log_format:
5136 case Builtin::BI__xray_customevent: {
5149 auto FTy = F->getFunctionType();
5150 auto Arg0 = E->
getArg(0);
5152 auto Arg0Ty = Arg0->getType();
5153 auto PTy0 = FTy->getParamType(0);
5154 if (PTy0 != Arg0Val->getType()) {
5155 if (Arg0Ty->isArrayType())
5158 Arg0Val = Builder.CreatePointerCast(Arg0Val, PTy0);
5161 auto PTy1 = FTy->getParamType(1);
5162 if (PTy1 != Arg1->getType())
5163 Arg1 = Builder.CreateTruncOrBitCast(Arg1, PTy1);
5164 return RValue::get(Builder.CreateCall(F, {Arg0Val, Arg1}));
5167 case Builtin::BI__xray_typedevent: {
5183 auto FTy = F->getFunctionType();
5185 auto PTy0 = FTy->getParamType(0);
5186 if (PTy0 != Arg0->getType())
5187 Arg0 = Builder.CreateTruncOrBitCast(Arg0, PTy0);
5188 auto Arg1 = E->
getArg(1);
5190 auto Arg1Ty = Arg1->getType();
5191 auto PTy1 = FTy->getParamType(1);
5192 if (PTy1 != Arg1Val->getType()) {
5193 if (Arg1Ty->isArrayType())
5196 Arg1Val = Builder.CreatePointerCast(Arg1Val, PTy1);
5199 auto PTy2 = FTy->getParamType(2);
5200 if (PTy2 != Arg2->getType())
5201 Arg2 = Builder.CreateTruncOrBitCast(Arg2, PTy2);
5202 return RValue::get(Builder.CreateCall(F, {Arg0, Arg1Val, Arg2}));
5205 case Builtin::BI__builtin_ms_va_start:
5206 case Builtin::BI__builtin_ms_va_end:
5209 BuiltinID == Builtin::BI__builtin_ms_va_start));
5211 case Builtin::BI__builtin_ms_va_copy: {
5228 Value *ArgPtr = Builder.CreateLoad(SrcAddr,
"ap.val");
5229 return RValue::get(Builder.CreateStore(ArgPtr, DestAddr));
5232 case Builtin::BI__builtin_get_device_side_mangled_name: {
5236 llvm::Constant *Zeros[] = {llvm::ConstantInt::get(
SizeTy, 0),
5237 llvm::ConstantInt::get(
SizeTy, 0)};
5238 auto *Ptr = llvm::ConstantExpr::getGetElementPtr(Str.getElementType(),
5239 Str.getPointer(), Zeros);
5265 LargestVectorWidth = std::max(LargestVectorWidth, VectorWidth);
5269 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
5271 llvm::Triple::getArchTypePrefix(
getTarget().getTriple().getArch());
5272 if (!Prefix.empty()) {
5273 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(Prefix.data(), Name);
5277 if (IntrinsicID == Intrinsic::not_intrinsic)
5278 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(), Name);
5281 if (IntrinsicID != Intrinsic::not_intrinsic) {
5286 unsigned ICEArguments = 0;
5292 llvm::FunctionType *FTy = F->getFunctionType();
5294 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; ++i) {
5297 if ((ICEArguments & (1 << i)) == 0) {
5302 ArgValue = llvm::ConstantInt::get(
5309 llvm::Type *PTy = FTy->getParamType(i);
5310 if (PTy != ArgValue->getType()) {
5312 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(PTy)) {
5313 if (PtrTy->getAddressSpace() !=
5314 ArgValue->getType()->getPointerAddressSpace()) {
5315 ArgValue = Builder.CreateAddrSpaceCast(
5317 ArgValue->getType()->getPointerTo(PtrTy->getAddressSpace()));
5321 assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) &&
5322 "Must be able to losslessly bit cast to param");
5325 if (PTy->isX86_AMXTy())
5326 ArgValue = Builder.CreateIntrinsic(Intrinsic::x86_cast_vector_to_tile,
5327 {ArgValue->getType()}, {ArgValue});
5329 ArgValue = Builder.CreateBitCast(ArgValue, PTy);
5332 Args.push_back(ArgValue);
5335 Value *
V = Builder.CreateCall(F, Args);
5338 llvm::Type *RetTy =
VoidTy;
5342 if (RetTy !=
V->getType()) {
5344 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(RetTy)) {
5345 if (PtrTy->getAddressSpace() !=
V->getType()->getPointerAddressSpace()) {
5346 V = Builder.CreateAddrSpaceCast(
5347 V,
V->getType()->getPointerTo(PtrTy->getAddressSpace()));
5351 assert(
V->getType()->canLosslesslyBitCastTo(RetTy) &&
5352 "Must be able to losslessly bit cast result type");
5355 if (
V->getType()->isX86_AMXTy())
5356 V = Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector, {RetTy},
5359 V = Builder.CreateBitCast(
V, RetTy);
5384 llvm_unreachable(
"No current target builtin returns complex");
5386 llvm_unreachable(
"Bad evaluation kind in EmitBuiltinExpr");
5396 unsigned BuiltinID,
const CallExpr *E,
5398 llvm::Triple::ArchType Arch) {
5400 case llvm::Triple::arm:
5401 case llvm::Triple::armeb:
5402 case llvm::Triple::thumb:
5403 case llvm::Triple::thumbeb:
5405 case llvm::Triple::aarch64:
5406 case llvm::Triple::aarch64_32:
5407 case llvm::Triple::aarch64_be:
5409 case llvm::Triple::bpfeb:
5410 case llvm::Triple::bpfel:
5412 case llvm::Triple::x86:
5413 case llvm::Triple::x86_64:
5415 case llvm::Triple::ppc:
5416 case llvm::Triple::ppcle:
5417 case llvm::Triple::ppc64:
5418 case llvm::Triple::ppc64le:
5420 case llvm::Triple::r600:
5421 case llvm::Triple::amdgcn:
5423 case llvm::Triple::systemz:
5425 case llvm::Triple::nvptx:
5426 case llvm::Triple::nvptx64:
5428 case llvm::Triple::wasm32:
5429 case llvm::Triple::wasm64:
5431 case llvm::Triple::hexagon:
5433 case llvm::Triple::riscv32:
5434 case llvm::Triple::riscv64:
5445 assert(
getContext().getAuxTargetInfo() &&
"Missing aux target info");
5457 bool HasLegalHalfType =
true,
5459 bool AllowBFloatArgsAndRet =
true) {
5460 int IsQuad = TypeFlags.
isQuad();
5464 return llvm::FixedVectorType::get(CGF->
Int8Ty, V1Ty ? 1 : (8 << IsQuad));
5467 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
5469 if (AllowBFloatArgsAndRet)
5470 return llvm::FixedVectorType::get(CGF->
BFloatTy, V1Ty ? 1 : (4 << IsQuad));
5472 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
5474 if (HasLegalHalfType)
5475 return llvm::FixedVectorType::get(CGF->
HalfTy, V1Ty ? 1 : (4 << IsQuad));
5477 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
5479 return llvm::FixedVectorType::get(CGF->
Int32Ty, V1Ty ? 1 : (2 << IsQuad));
5482 return llvm::FixedVectorType::get(CGF->
Int64Ty, V1Ty ? 1 : (1 << IsQuad));
5487 return llvm::FixedVectorType::get(CGF->
Int8Ty, 16);
5489 return llvm::FixedVectorType::get(CGF->
FloatTy, V1Ty ? 1 : (2 << IsQuad));
5491 return llvm::FixedVectorType::get(CGF->
DoubleTy, V1Ty ? 1 : (1 << IsQuad));
5493 llvm_unreachable(
"Unknown vector element type!");
5498 int IsQuad = IntTypeFlags.
isQuad();
5501 return llvm::FixedVectorType::get(CGF->
HalfTy, (4 << IsQuad));
5503 return llvm::FixedVectorType::get(CGF->
FloatTy, (2 << IsQuad));
5505 return llvm::FixedVectorType::get(CGF->
DoubleTy, (1 << IsQuad));
5507 llvm_unreachable(
"Type can't be converted to floating-point!");
5512 const ElementCount &Count) {
5513 Value *SV = llvm::ConstantVector::getSplat(Count,
C);
5514 return Builder.CreateShuffleVector(
V,
V, SV,
"lane");
5524 unsigned shift,
bool rightshift) {
5526 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
5527 ai != ae; ++ai, ++j) {
5528 if (F->isConstrainedFPIntrinsic())
5529 if (ai->getType()->isMetadataTy())
5531 if (shift > 0 && shift == j)
5534 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name);
5537 if (F->isConstrainedFPIntrinsic())
5538 return Builder.CreateConstrainedFPCall(F, Ops, name);
5540 return Builder.CreateCall(F, Ops, name);
5546 return ConstantInt::get(Ty, neg ? -SV : SV);
5551 llvm::Type *Ty,
bool usgn,
5556 int EltSize = VTy->getScalarSizeInBits();
5558 Vec = Builder.CreateBitCast(Vec, Ty);
5562 if (ShiftAmt == EltSize) {
5565 return llvm::ConstantAggregateZero::get(VTy);
5570 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
5576 return Builder.CreateLShr(Vec, Shift, name);
5578 return Builder.CreateAShr(Vec, Shift, name);
5604struct ARMVectorIntrinsicInfo {
5605 const char *NameHint;
5607 unsigned LLVMIntrinsic;
5608 unsigned AltLLVMIntrinsic;
5611 bool operator<(
unsigned RHSBuiltinID)
const {
5612 return BuiltinID < RHSBuiltinID;
5614 bool operator<(
const ARMVectorIntrinsicInfo &TE)
const {
5615 return BuiltinID < TE.BuiltinID;
5620#define NEONMAP0(NameBase) \
5621 { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 }
5623#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
5624 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
5625 Intrinsic::LLVMIntrinsic, 0, TypeModifier }
5627#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
5628 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
5629 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
5633 NEONMAP1(__a32_vcvt_bf16_v, arm_neon_vcvtfp2bf, 0),
5640 NEONMAP1(vabs_v, arm_neon_vabs, 0),
5641 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
5645 NEONMAP1(vaesdq_v, arm_neon_aesd, 0),
5646 NEONMAP1(vaeseq_v, arm_neon_aese, 0),
5647 NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0),
5648 NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0),
5649 NEONMAP1(vbfdot_v, arm_neon_bfdot, 0),
5650 NEONMAP1(vbfdotq_v, arm_neon_bfdot, 0),
5651 NEONMAP1(vbfmlalbq_v, arm_neon_bfmlalb, 0),
5652 NEONMAP1(vbfmlaltq_v, arm_neon_bfmlalt, 0),
5653 NEONMAP1(vbfmmlaq_v, arm_neon_bfmmla, 0),
5660 NEONMAP1(vcage_v, arm_neon_vacge, 0),
5661 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
5662 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
5663 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
5664 NEONMAP1(vcale_v, arm_neon_vacge, 0),
5665 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
5666 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
5667 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
5684 NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
5686 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
5688 NEONMAP2(vcvt_n_f16_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
5689 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
5690 NEONMAP1(vcvt_n_s16_v, arm_neon_vcvtfp2fxs, 0),
5691 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
5692 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
5693 NEONMAP1(vcvt_n_u16_v, arm_neon_vcvtfp2fxu, 0),
5694 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
5695 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
5702 NEONMAP1(vcvta_s16_v, arm_neon_vcvtas, 0),
5703 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
5704 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
5705 NEONMAP1(vcvta_u16_v, arm_neon_vcvtau, 0),
5706 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
5707 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
5708 NEONMAP1(vcvtaq_s16_v, arm_neon_vcvtas, 0),
5709 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
5710 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
5711 NEONMAP1(vcvtaq_u16_v, arm_neon_vcvtau, 0),
5712 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
5713 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
5714 NEONMAP1(vcvth_bf16_f32, arm_neon_vcvtbfp2bf, 0),
5715 NEONMAP1(vcvtm_s16_v, arm_neon_vcvtms, 0),
5716 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
5717 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
5718 NEONMAP1(vcvtm_u16_v, arm_neon_vcvtmu, 0),
5719 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
5720 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
5721 NEONMAP1(vcvtmq_s16_v, arm_neon_vcvtms, 0),
5722 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
5723 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
5724 NEONMAP1(vcvtmq_u16_v, arm_neon_vcvtmu, 0),
5725 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
5726 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
5727 NEONMAP1(vcvtn_s16_v, arm_neon_vcvtns, 0),
5728 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
5729 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
5730 NEONMAP1(vcvtn_u16_v, arm_neon_vcvtnu, 0),
5731 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
5732 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
5733 NEONMAP1(vcvtnq_s16_v, arm_neon_vcvtns, 0),
5734 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
5735 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
5736 NEONMAP1(vcvtnq_u16_v, arm_neon_vcvtnu, 0),
5737 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
5738 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
5739 NEONMAP1(vcvtp_s16_v, arm_neon_vcvtps, 0),
5740 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
5741 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
5742 NEONMAP1(vcvtp_u16_v, arm_neon_vcvtpu, 0),
5743 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
5744 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
5745 NEONMAP1(vcvtpq_s16_v, arm_neon_vcvtps, 0),
5746 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
5747 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
5748 NEONMAP1(vcvtpq_u16_v, arm_neon_vcvtpu, 0),
5749 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
5750 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
5753 NEONMAP2(vcvtq_n_f16_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
5754 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
5755 NEONMAP1(vcvtq_n_s16_v, arm_neon_vcvtfp2fxs, 0),
5756 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
5757 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
5758 NEONMAP1(vcvtq_n_u16_v, arm_neon_vcvtfp2fxu, 0),
5759 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
5760 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
5767 NEONMAP2(vdot_v, arm_neon_udot, arm_neon_sdot, 0),
5768 NEONMAP2(vdotq_v, arm_neon_udot, arm_neon_sdot, 0),
5778 NEONMAP1(vld1_v, arm_neon_vld1, 0),
5779 NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
5780 NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
5781 NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
5783 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
5784 NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
5785 NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
5786 NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
5787 NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
5788 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
5789 NEONMAP1(vld2_v, arm_neon_vld2, 0),
5790 NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
5791 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
5792 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
5793 NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
5794 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
5795 NEONMAP1(vld3_v, arm_neon_vld3, 0),
5796 NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
5797 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
5798 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
5799 NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
5800 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
5801 NEONMAP1(vld4_v, arm_neon_vld4, 0),
5802 NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
5803 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
5804 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
5813 NEONMAP2(vmmlaq_v, arm_neon_ummla, arm_neon_smmla, 0),
5831 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, sadd_sat, 0),
5832 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, ssub_sat, 0),
5852 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
5853 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
5857 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
5858 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
5881 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
5882 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
5886 NEONMAP1(vsha1su0q_v, arm_neon_sha1su0, 0),
5887 NEONMAP1(vsha1su1q_v, arm_neon_sha1su1, 0),
5888 NEONMAP1(vsha256h2q_v, arm_neon_sha256h2, 0),
5889 NEONMAP1(vsha256hq_v, arm_neon_sha256h, 0),
5890 NEONMAP1(vsha256su0q_v, arm_neon_sha256su0, 0),
5891 NEONMAP1(vsha256su1q_v, arm_neon_sha256su1, 0),
5900 NEONMAP1(vst1_v, arm_neon_vst1, 0),
5901 NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
5902 NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
5903 NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
5904 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
5905 NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
5906 NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
5907 NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
5908 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
5909 NEONMAP1(vst2_v, arm_neon_vst2, 0),
5910 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
5911 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
5912 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
5913 NEONMAP1(vst3_v, arm_neon_vst3, 0),
5914 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
5915 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
5916 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
5917 NEONMAP1(vst4_v, arm_neon_vst4, 0),
5918 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
5919 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
5925 NEONMAP1(vusdot_v, arm_neon_usdot, 0),
5926 NEONMAP1(vusdotq_v, arm_neon_usdot, 0),
5927 NEONMAP1(vusmmlaq_v, arm_neon_usmmla, 0),
5935 NEONMAP1(__a64_vcvtq_low_bf16_v, aarch64_neon_bfcvtn, 0),
5940 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
5941 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
5946 NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0),
5947 NEONMAP1(vaeseq_v, aarch64_crypto_aese, 0),
5948 NEONMAP1(vaesimcq_v, aarch64_crypto_aesimc, 0),
5949 NEONMAP1(vaesmcq_v, aarch64_crypto_aesmc, 0),
5951 NEONMAP1(vbfdot_v, aarch64_neon_bfdot, 0),
5952 NEONMAP1(vbfdotq_v, aarch64_neon_bfdot, 0),
5953 NEONMAP1(vbfmlalbq_v, aarch64_neon_bfmlalb, 0),
5954 NEONMAP1(vbfmlaltq_v, aarch64_neon_bfmlalt, 0),
5955 NEONMAP1(vbfmmlaq_v, aarch64_neon_bfmmla, 0),
5960 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
5961 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
5962 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
5963 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
5964 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
5965 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
5966 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
5967 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
5992 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
5994 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
5996 NEONMAP2(vcvt_n_f16_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
5997 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
5998 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
5999 NEONMAP1(vcvt_n_s16_v, aarch64_neon_vcvtfp2fxs, 0),
6000 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
6001 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
6002 NEONMAP1(vcvt_n_u16_v, aarch64_neon_vcvtfp2fxu, 0),
6003 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
6004 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
6007 NEONMAP1(vcvtq_high_bf16_v, aarch64_neon_bfcvtn2, 0),
6008 NEONMAP2(vcvtq_n_f16_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6009 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6010 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6011 NEONMAP1(vcvtq_n_s16_v, aarch64_neon_vcvtfp2fxs, 0),
6012 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
6013 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
6014 NEONMAP1(vcvtq_n_u16_v, aarch64_neon_vcvtfp2fxu, 0),
6015 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
6016 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
6018 NEONMAP2(vdot_v, aarch64_neon_udot, aarch64_neon_sdot, 0),
6019 NEONMAP2(vdotq_v, aarch64_neon_udot, aarch64_neon_sdot, 0),
6025 NEONMAP1(vfmlal_high_v, aarch64_neon_fmlal2, 0),
6026 NEONMAP1(vfmlal_low_v, aarch64_neon_fmlal, 0),
6027 NEONMAP1(vfmlalq_high_v, aarch64_neon_fmlal2, 0),
6028 NEONMAP1(vfmlalq_low_v, aarch64_neon_fmlal, 0),
6029 NEONMAP1(vfmlsl_high_v, aarch64_neon_fmlsl2, 0),
6030 NEONMAP1(vfmlsl_low_v, aarch64_neon_fmlsl, 0),
6031 NEONMAP1(vfmlslq_high_v, aarch64_neon_fmlsl2, 0),
6032 NEONMAP1(vfmlslq_low_v, aarch64_neon_fmlsl, 0),
6037 NEONMAP1(vld1_x2_v, aarch64_neon_ld1x2, 0),
6038 NEONMAP1(vld1_x3_v, aarch64_neon_ld1x3, 0),
6039 NEONMAP1(vld1_x4_v, aarch64_neon_ld1x4, 0),
6040 NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
6041 NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
6042 NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
6043 NEONMAP2(vmmlaq_v, aarch64_neon_ummla, aarch64_neon_smmla, 0),
6056 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
6057 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
6058 NEONMAP1(vqdmulh_lane_v, aarch64_neon_sqdmulh_lane, 0),
6059 NEONMAP1(vqdmulh_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
6061 NEONMAP1(vqdmulhq_lane_v, aarch64_neon_sqdmulh_lane, 0),
6062 NEONMAP1(vqdmulhq_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
6073 NEONMAP1(vqrdmulh_lane_v, aarch64_neon_sqrdmulh_lane, 0),
6074 NEONMAP1(vqrdmulh_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
6076 NEONMAP1(vqrdmulhq_lane_v, aarch64_neon_sqrdmulh_lane, 0),
6077 NEONMAP1(vqrdmulhq_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
6085 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
6086 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
6090 NEONMAP1(vrax1q_v, aarch64_crypto_rax1, 0),
6091 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
6092 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
6111 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
6112 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
6116 NEONMAP1(vsha1su0q_v, aarch64_crypto_sha1su0, 0),
6117 NEONMAP1(vsha1su1q_v, aarch64_crypto_sha1su1, 0),
6118 NEONMAP1(vsha256h2q_v, aarch64_crypto_sha256h2, 0),
6119 NEONMAP1(vsha256hq_v, aarch64_crypto_sha256h, 0),
6120 NEONMAP1(vsha256su0q_v, aarch64_crypto_sha256su0, 0),
6121 NEONMAP1(vsha256su1q_v, aarch64_crypto_sha256su1, 0),
6122 NEONMAP1(vsha512h2q_v, aarch64_crypto_sha512h2, 0),
6123 NEONMAP1(vsha512hq_v, aarch64_crypto_sha512h, 0),
6124 NEONMAP1(vsha512su0q_v, aarch64_crypto_sha512su0, 0),
6125 NEONMAP1(vsha512su1q_v, aarch64_crypto_sha512su1, 0),
6134 NEONMAP1(vsm3partw1q_v, aarch64_crypto_sm3partw1, 0),
6135 NEONMAP1(vsm3partw2q_v, aarch64_crypto_sm3partw2, 0),
6136 NEONMAP1(vsm3ss1q_v, aarch64_crypto_sm3ss1, 0),
6137 NEONMAP1(vsm3tt1aq_v, aarch64_crypto_sm3tt1a, 0),
6138 NEONMAP1(vsm3tt1bq_v, aarch64_crypto_sm3tt1b, 0),
6139 NEONMAP1(vsm3tt2aq_v, aarch64_crypto_sm3tt2a, 0),
6140 NEONMAP1(vsm3tt2bq_v, aarch64_crypto_sm3tt2b, 0),
6141 NEONMAP1(vsm4ekeyq_v, aarch64_crypto_sm4ekey, 0),
6142 NEONMAP1(vsm4eq_v, aarch64_crypto_sm4e, 0),
6143 NEONMAP1(vst1_x2_v, aarch64_neon_st1x2, 0),
6144 NEONMAP1(vst1_x3_v, aarch64_neon_st1x3, 0),
6145 NEONMAP1(vst1_x4_v, aarch64_neon_st1x4, 0),
6146 NEONMAP1(vst1q_x2_v, aarch64_neon_st1x2, 0),
6147 NEONMAP1(vst1q_x3_v, aarch64_neon_st1x3, 0),
6148 NEONMAP1(vst1q_x4_v, aarch64_neon_st1x4, 0),
6152 NEONMAP1(vusdot_v, aarch64_neon_usdot, 0),
6153 NEONMAP1(vusdotq_v, aarch64_neon_usdot, 0),
6154 NEONMAP1(vusmmlaq_v, aarch64_neon_usmmla, 0),
6155 NEONMAP1(vxarq_v, aarch64_crypto_xar, 0),
6193 NEONMAP1(vcvth_bf16_f32, aarch64_neon_bfcvt, 0),
6212 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
6233 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
6261 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
6342 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
6343 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
6344 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
6345 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
6401#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
6403 #NameBase, SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
6407#define SVEMAP2(NameBase, TypeModifier) \
6408 { #NameBase, SVE::BI__builtin_sve_##NameBase, 0, 0, TypeModifier }
6410#define GET_SVE_LLVM_INTRINSIC_MAP
6411#include "clang/Basic/arm_sve_builtin_cg.inc"
6412#include "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
6413#undef GET_SVE_LLVM_INTRINSIC_MAP
6425static const ARMVectorIntrinsicInfo *
6427 unsigned BuiltinID,
bool &MapProvenSorted) {
6430 if (!MapProvenSorted) {
6431 assert(llvm::is_sorted(IntrinsicMap));
6432 MapProvenSorted =
true;
6436 const ARMVectorIntrinsicInfo *Builtin =
6437 llvm::lower_bound(IntrinsicMap, BuiltinID);
6439 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
6447 llvm::Type *ArgType,
6460 Ty = llvm::FixedVectorType::get(
6461 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
6468 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
6469 ArgType = llvm::FixedVectorType::get(ArgType, Elts);
6473 Tys.push_back(ArgType);
6476 Tys.push_back(ArgType);
6487 unsigned BuiltinID = SISDInfo.BuiltinID;
6488 unsigned int Int = SISDInfo.LLVMIntrinsic;
6489 unsigned Modifier = SISDInfo.TypeModifier;
6490 const char *
s = SISDInfo.NameHint;
6492 switch (BuiltinID) {
6493 case NEON::BI__builtin_neon_vcled_s64:
6494 case NEON::BI__builtin_neon_vcled_u64:
6495 case NEON::BI__builtin_neon_vcles_f32:
6496 case NEON::BI__builtin_neon_vcled_f64:
6497 case NEON::BI__builtin_neon_vcltd_s64:
6498 case NEON::BI__builtin_neon_vcltd_u64:
6499 case NEON::BI__builtin_neon_vclts_f32:
6500 case NEON::BI__builtin_neon_vcltd_f64:
6501 case NEON::BI__builtin_neon_vcales_f32:
6502 case NEON::BI__builtin_neon_vcaled_f64:
6503 case NEON::BI__builtin_neon_vcalts_f32:
6504 case NEON::BI__builtin_neon_vcaltd_f64:
6508 std::swap(Ops[0], Ops[1]);
6512 assert(Int &&
"Generic code assumes a valid intrinsic");
6520 ConstantInt *C0 = ConstantInt::get(CGF.
SizeTy, 0);
6521 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
6522 ai != ae; ++ai, ++j) {
6523 llvm::Type *ArgTy = ai->getType();
6524 if (Ops[j]->
getType()->getPrimitiveSizeInBits() ==
6525 ArgTy->getPrimitiveSizeInBits())
6528 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
6531 Ops[j] = CGF.
Builder.CreateTruncOrBitCast(
6532 Ops[j], cast<llvm::VectorType>(ArgTy)->getElementType());
6534 CGF.
Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0);
6539 if (ResultType->getPrimitiveSizeInBits().getFixedSize() <
6540 Result->getType()->getPrimitiveSizeInBits().getFixedSize())
6547 unsigned BuiltinID,
unsigned LLVMIntrinsic,
unsigned AltLLVMIntrinsic,
6548 const char *NameHint,
unsigned Modifier,
const CallExpr *E,
6550 llvm::Triple::ArchType Arch) {
6560 bool Usgn =
Type.isUnsigned();
6561 bool Quad =
Type.isQuad();
6563 const bool AllowBFloatArgsAndRet =
6566 llvm::FixedVectorType *VTy =
6567 GetNeonType(
this,
Type, HasLegalHalfType,
false, AllowBFloatArgsAndRet);
6568 llvm::Type *Ty = VTy;
6572 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
6573 return Builder.getInt32(addr.getAlignment().getQuantity());
6576 unsigned Int = LLVMIntrinsic;
6578 Int = AltLLVMIntrinsic;
6580 switch (BuiltinID) {
6582 case NEON::BI__builtin_neon_splat_lane_v:
6583 case NEON::BI__builtin_neon_splat_laneq_v:
6584 case NEON::BI__builtin_neon_splatq_lane_v:
6585 case NEON::BI__builtin_neon_splatq_laneq_v: {
6586 auto NumElements = VTy->getElementCount();
6587 if (BuiltinID == NEON::BI__builtin_neon_splatq_lane_v)
6588 NumElements = NumElements * 2;
6589 if (BuiltinID == NEON::BI__builtin_neon_splat_laneq_v)
6590 NumElements = NumElements.divideCoefficientBy(2);
6592 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
6593 return EmitNeonSplat(Ops[0], cast<ConstantInt>(Ops[1]), NumElements);
6595 case NEON::BI__builtin_neon_vpadd_v:
6596 case NEON::BI__builtin_neon_vpaddq_v:
6598 if (VTy->getElementType()->isFloatingPointTy() &&
6599 Int == Intrinsic::aarch64_neon_addp)
6600 Int = Intrinsic::aarch64_neon_faddp;
6602 case NEON::BI__builtin_neon_vabs_v:
6603 case NEON::BI__builtin_neon_vabsq_v:
6604 if (VTy->getElementType()->isFloatingPointTy())
6607 case NEON::BI__builtin_neon_vadd_v:
6608 case NEON::BI__builtin_neon_vaddq_v: {
6609 llvm::Type *VTy = llvm::FixedVectorType::get(
Int8Ty, Quad ? 16 : 8);
6610 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
6611 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
6612 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]);
6613 return Builder.CreateBitCast(Ops[0], Ty);
6615 case NEON::BI__builtin_neon_vaddhn_v: {
6616 llvm::FixedVectorType *SrcTy =
6617 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
6620 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
6621 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
6622 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1],
"vaddhn");
6625 Constant *ShiftAmt =
6626 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
6627 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt,
"vaddhn");
6630 return Builder.CreateTrunc(Ops[0], VTy,
"vaddhn");
6632 case NEON::BI__builtin_neon_vcale_v:
6633 case NEON::BI__builtin_neon_vcaleq_v:
6634 case NEON::BI__builtin_neon_vcalt_v:
6635 case NEON::BI__builtin_neon_vcaltq_v:
6636 std::swap(Ops[0], Ops[1]);
6638 case NEON::BI__builtin_neon_vcage_v:
6639 case NEON::BI__builtin_neon_vcageq_v:
6640 case NEON::BI__builtin_neon_vcagt_v:
6641 case NEON::BI__builtin_neon_vcagtq_v: {
6643 switch (VTy->getScalarSizeInBits()) {
6644 default: llvm_unreachable(
"unexpected type");
6655 auto *VecFlt = llvm::FixedVectorType::get(Ty, VTy->getNumElements());
6656 llvm::Type *Tys[] = { VTy, VecFlt };
6660 case NEON::BI__builtin_neon_vceqz_v:
6661 case NEON::BI__builtin_neon_vceqzq_v:
6663 ICmpInst::ICMP_EQ,
"vceqz");
6664 case NEON::BI__builtin_neon_vcgez_v:
6665 case NEON::BI__builtin_neon_vcgezq_v:
6667 ICmpInst::ICMP_SGE,
"vcgez");
6668 case NEON::BI__builtin_neon_vclez_v:
6669 case NEON::BI__builtin_neon_vclezq_v:
6671 ICmpInst::ICMP_SLE,
"vclez");
6672 case NEON::BI__builtin_neon_vcgtz_v:
6673 case NEON::BI__builtin_neon_vcgtzq_v:
6675 ICmpInst::ICMP_SGT,
"vcgtz");
6676 case NEON::BI__builtin_neon_vcltz_v:
6677 case NEON::BI__builtin_neon_vcltzq_v:
6679 ICmpInst::ICMP_SLT,
"vcltz");
6680 case NEON::BI__builtin_neon_vclz_v:
6681 case NEON::BI__builtin_neon_vclzq_v:
6684 Ops.push_back(Builder.getInt1(
getTarget().isCLZForZeroUndef()));
6686 case NEON::BI__builtin_neon_vcvt_f32_v:
6687 case NEON::BI__builtin_neon_vcvtq_f32_v:
6688 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6691 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
6692 : Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
6693 case NEON::BI__builtin_neon_vcvt_f16_v:
6694 case NEON::BI__builtin_neon_vcvtq_f16_v:
6695 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6698 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
6699 : Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
6700 case NEON::BI__builtin_neon_vcvt_n_f16_v:
6701 case NEON::BI__builtin_neon_vcvt_n_f32_v:
6702 case NEON::BI__builtin_neon_vcvt_n_f64_v:
6703 case NEON::BI__builtin_neon_vcvtq_n_f16_v:
6704 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
6705 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
6707 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
6711 case NEON::BI__builtin_neon_vcvt_n_s16_v:
6712 case NEON::BI__builtin_neon_vcvt_n_s32_v:
6713 case NEON::BI__builtin_neon_vcvt_n_u16_v:
6714 case NEON::BI__builtin_neon_vcvt_n_u32_v:
6715 case NEON::BI__builtin_neon_vcvt_n_s64_v:
6716 case NEON::BI__builtin_neon_vcvt_n_u64_v:
6717 case NEON::BI__builtin_neon_vcvtq_n_s16_v:
6718 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
6719 case NEON::BI__builtin_neon_vcvtq_n_u16_v:
6720 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
6721 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
6722 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
6727 case NEON::BI__builtin_neon_vcvt_s32_v:
6728 case NEON::BI__builtin_neon_vcvt_u32_v:
6729 case NEON::BI__builtin_neon_vcvt_s64_v:
6730 case NEON::BI__builtin_neon_vcvt_u64_v:
6731 case NEON::BI__builtin_neon_vcvt_s16_v:
6732 case NEON::BI__builtin_neon_vcvt_u16_v:
6733 case NEON::BI__builtin_neon_vcvtq_s32_v:
6734 case NEON::BI__builtin_neon_vcvtq_u32_v:
6735 case NEON::BI__builtin_neon_vcvtq_s64_v:
6736 case NEON::BI__builtin_neon_vcvtq_u64_v:
6737 case NEON::BI__builtin_neon_vcvtq_s16_v:
6738 case NEON::BI__builtin_neon_vcvtq_u16_v: {
6740 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty,
"vcvt")
6741 : Builder.CreateFPToSI(Ops[0], Ty,
"vcvt");
6743 case NEON::BI__builtin_neon_vcvta_s16_v:
6744 case NEON::BI__builtin_neon_vcvta_s32_v:
6745 case NEON::BI__builtin_neon_vcvta_s64_v:
6746 case NEON::BI__builtin_neon_vcvta_u16_v:
6747 case NEON::BI__builtin_neon_vcvta_u32_v:
6748 case NEON::BI__builtin_neon_vcvta_u64_v:
6749 case NEON::BI__builtin_neon_vcvtaq_s16_v:
6750 case NEON::BI__builtin_neon_vcvtaq_s32_v:
6751 case NEON::BI__builtin_neon_vcvtaq_s64_v:
6752 case NEON::BI__builtin_neon_vcvtaq_u16_v:
6753 case NEON::BI__builtin_neon_vcvtaq_u32_v:
6754 case NEON::BI__builtin_neon_vcvtaq_u64_v:
6755 case NEON::BI__builtin_neon_vcvtn_s16_v:
6756 case NEON::BI__builtin_neon_vcvtn_s32_v:
6757 case NEON::BI__builtin_neon_vcvtn_s64_v:
6758 case NEON::BI__builtin_neon_vcvtn_u16_v:
6759 case NEON::BI__builtin_neon_vcvtn_u32_v:
6760 case NEON::BI__builtin_neon_vcvtn_u64_v:
6761 case NEON::BI__builtin_neon_vcvtnq_s16_v:
6762 case NEON::BI__builtin_neon_vcvtnq_s32_v:
6763 case NEON::BI__builtin_neon_vcvtnq_s64_v:
6764 case NEON::BI__builtin_neon_vcvtnq_u16_v:
6765 case NEON::BI__builtin_neon_vcvtnq_u32_v:
6766 case NEON::BI__builtin_neon_vcvtnq_u64_v:
6767 case NEON::BI__builtin_neon_vcvtp_s16_v:
6768 case NEON::BI__builtin_neon_vcvtp_s32_v:
6769 case NEON::BI__builtin_neon_vcvtp_s64_v:
6770 case NEON::BI__builtin_neon_vcvtp_u16_v:
6771 case NEON::BI__builtin_neon_vcvtp_u32_v:
6772 case NEON::BI__builtin_neon_vcvtp_u64_v:
6773 case NEON::BI__builtin_neon_vcvtpq_s16_v:
6774 case NEON::BI__builtin_neon_vcvtpq_s32_v:
6775 case NEON::BI__builtin_neon_vcvtpq_s64_v:
6776 case NEON::BI__builtin_neon_vcvtpq_u16_v:
6777 case NEON::BI__builtin_neon_vcvtpq_u32_v:
6778 case NEON::BI__builtin_neon_vcvtpq_u64_v:
6779 case NEON::BI__builtin_neon_vcvtm_s16_v:
6780 case NEON::BI__builtin_neon_vcvtm_s32_v:
6781 case NEON::BI__builtin_neon_vcvtm_s64_v:
6782 case NEON::BI__builtin_neon_vcvtm_u16_v:
6783 case NEON::BI__builtin_neon_vcvtm_u32_v:
6784 case NEON::BI__builtin_neon_vcvtm_u64_v:
6785 case NEON::BI__builtin_neon_vcvtmq_s16_v:
6786 case NEON::BI__builtin_neon_vcvtmq_s32_v:
6787 case NEON::BI__builtin_neon_vcvtmq_s64_v:
6788 case NEON::BI__builtin_neon_vcvtmq_u16_v:
6789 case NEON::BI__builtin_neon_vcvtmq_u32_v:
6790 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
6794 case NEON::BI__builtin_neon_vcvtx_f32_v: {
6795 llvm::Type *Tys[2] = { VTy->getTruncatedElementVectorType(VTy), Ty};
6799 case NEON::BI__builtin_neon_vext_v:
6800 case NEON::BI__builtin_neon_vextq_v: {
6803 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
6804 Indices.push_back(i+CV);
6806 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6807 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6808 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices,
"vext");
6810 case NEON::BI__builtin_neon_vfma_v:
6811 case NEON::BI__builtin_neon_vfmaq_v: {
6812 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6813 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6814 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6818 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
6819 {Ops[1], Ops[2], Ops[0]});
6821 case NEON::BI__builtin_neon_vld1_v:
6822 case NEON::BI__builtin_neon_vld1q_v: {
6824 Ops.push_back(getAlignmentValue32(PtrOp0));
6827 case NEON::BI__builtin_neon_vld1_x2_v:
6828 case NEON::BI__builtin_neon_vld1q_x2_v:
6829 case NEON::BI__builtin_neon_vld1_x3_v:
6830 case NEON::BI__builtin_neon_vld1q_x3_v:
6831 case NEON::BI__builtin_neon_vld1_x4_v:
6832 case NEON::BI__builtin_neon_vld1q_x4_v: {
6833 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getElementType());
6834 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
6835 llvm::Type *Tys[2] = { VTy, PTy };
6837 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld1xN");
6838 Ty = llvm::PointerType::getUnqual(Ops[1]->
getType());
6839 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6840 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6842 case NEON::BI__builtin_neon_vld2_v:
6843 case NEON::BI__builtin_neon_vld2q_v:
6844 case NEON::BI__builtin_neon_vld3_v:
6845 case NEON::BI__builtin_neon_vld3q_v:
6846 case NEON::BI__builtin_neon_vld4_v:
6847 case NEON::BI__builtin_neon_vld4q_v:
6848 case NEON::BI__builtin_neon_vld2_dup_v:
6849 case NEON::BI__builtin_neon_vld2q_dup_v:
6850 case NEON::BI__builtin_neon_vld3_dup_v:
6851 case NEON::BI__builtin_neon_vld3q_dup_v:
6852 case NEON::BI__builtin_neon_vld4_dup_v:
6853 case NEON::BI__builtin_neon_vld4q_dup_v: {
6856 Value *Align = getAlignmentValue32(PtrOp1);
6857 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint);
6858 Ty = llvm::PointerType::getUnqual(Ops[1]->
getType());
6859 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6860 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6862 case NEON::BI__builtin_neon_vld1_dup_v:
6863 case NEON::BI__builtin_neon_vld1q_dup_v: {
6864 Value *
V = UndefValue::get(Ty);
6865 PtrOp0 = Builder.CreateElementBitCast(PtrOp0, VTy->getElementType());
6866 LoadInst *Ld = Builder.CreateLoad(PtrOp0);
6867 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
6868 Ops[0] = Builder.CreateInsertElement(
V, Ld, CI);
6871 case NEON::BI__builtin_neon_vld2_lane_v:
6872 case NEON::BI__builtin_neon_vld2q_lane_v:
6873 case NEON::BI__builtin_neon_vld3_lane_v:
6874 case NEON::BI__builtin_neon_vld3q_lane_v:
6875 case NEON::BI__builtin_neon_vld4_lane_v:
6876 case NEON::BI__builtin_neon_vld4q_lane_v: {
6879 for (
unsigned I = 2; I < Ops.size() - 1; ++I)
6880 Ops[I] = Builder.CreateBitCast(Ops[I], Ty);
6881 Ops.push_back(getAlignmentValue32(PtrOp1));
6882 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint);
6883 Ty = llvm::PointerType::getUnqual(Ops[1]->
getType());
6884 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6885 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6887 case NEON::BI__builtin_neon_vmovl_v: {
6888 llvm::FixedVectorType *DTy =
6889 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
6890 Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
6892 return Builder.CreateZExt(Ops[0], Ty,
"vmovl");
6893 return Builder.CreateSExt(Ops[0], Ty,
"vmovl");
6895 case NEON::BI__builtin_neon_vmovn_v: {
6896 llvm::FixedVectorType *QTy =
6897 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
6898 Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
6899 return Builder.CreateTrunc(Ops[0], Ty,
"vmovn");
6901 case NEON::BI__builtin_neon_vmull_v:
6907 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
6908 Int =
Type.isPoly() ? (
unsigned)Intrinsic::arm_neon_vmullp : Int;
6910 case NEON::BI__builtin_neon_vpadal_v:
6911 case NEON::BI__builtin_neon_vpadalq_v: {
6913 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
6917 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
6918 llvm::Type *Tys[2] = { Ty, NarrowTy };
6921 case NEON::BI__builtin_neon_vpaddl_v:
6922 case NEON::BI__builtin_neon_vpaddlq_v: {
6924 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
6925 llvm::Type *EltTy = llvm::IntegerType::get(
getLLVMContext(), EltBits / 2);
6927 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
6928 llvm::Type *Tys[2] = { Ty, NarrowTy };
6931 case NEON::BI__builtin_neon_vqdmlal_v:
6932 case NEON::BI__builtin_neon_vqdmlsl_v: {
6939 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
6940 case NEON::BI__builtin_neon_vqdmulh_lane_v:
6941 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
6942 case NEON::BI__builtin_neon_vqrdmulh_lane_v: {
6944 if (BuiltinID == NEON::BI__builtin_neon_vqdmulhq_lane_v ||
6945 BuiltinID == NEON::BI__builtin_neon_vqrdmulhq_lane_v)
6946 RTy = llvm::FixedVectorType::get(RTy->getElementType(),
6947 RTy->getNumElements() * 2);
6948 llvm::Type *Tys[2] = {
6953 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
6954 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
6955 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
6956 case NEON::BI__builtin_neon_vqrdmulh_laneq_v: {
6957 llvm::Type *Tys[2] = {
6962 case NEON::BI__builtin_neon_vqshl_n_v:
6963 case NEON::BI__builtin_neon_vqshlq_n_v:
6966 case NEON::BI__builtin_neon_vqshlu_n_v:
6967 case NEON::BI__builtin_neon_vqshluq_n_v:
6970 case NEON::BI__builtin_neon_vrecpe_v:
6971 case NEON::BI__builtin_neon_vrecpeq_v:
6972 case NEON::BI__builtin_neon_vrsqrte_v:
6973 case NEON::BI__builtin_neon_vrsqrteq_v:
6974 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
6976 case NEON::BI__builtin_neon_vrndi_v:
6977 case NEON::BI__builtin_neon_vrndiq_v:
6978 Int = Builder.getIsFPConstrained()
6979 ? Intrinsic::experimental_constrained_nearbyint
6980 : Intrinsic::nearbyint;
6982 case NEON::BI__builtin_neon_vrshr_n_v:
6983 case NEON::BI__builtin_neon_vrshrq_n_v:
6986 case NEON::BI__builtin_neon_vsha512hq_v:
6987 case NEON::BI__builtin_neon_vsha512h2q_v:
6988 case NEON::BI__builtin_neon_vsha512su0q_v:
6989 case NEON::BI__builtin_neon_vsha512su1q_v: {
6993 case NEON::BI__builtin_neon_vshl_n_v:
6994 case NEON::BI__builtin_neon_vshlq_n_v:
6996 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1],
6998 case NEON::BI__builtin_neon_vshll_n_v: {
6999 llvm::FixedVectorType *SrcTy =
7000 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
7001 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
7003 Ops[0] = Builder.CreateZExt(Ops[0], VTy);
7005 Ops[0] = Builder.CreateSExt(Ops[0], VTy);
7007 return Builder.CreateShl(Ops[0], Ops[1],
"vshll_n");
7009 case NEON::BI__builtin_neon_vshrn_n_v: {
7010 llvm::FixedVectorType *SrcTy =
7011 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
7012 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
7015 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]);
7017 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]);
7018 return Builder.CreateTrunc(Ops[0], Ty,
"vshrn_n");
7020 case NEON::BI__builtin_neon_vshr_n_v:
7021 case NEON::BI__builtin_neon_vshrq_n_v:
7023 case NEON::BI__builtin_neon_vst1_v:
7024 case NEON::BI__builtin_neon_vst1q_v:
7025 case NEON::BI__builtin_neon_vst2_v:
7026 case NEON::BI__builtin_neon_vst2q_v:
7027 case NEON::BI__builtin_neon_vst3_v:
7028 case NEON::BI__builtin_neon_vst3q_v:
7029 case NEON::BI__builtin_neon_vst4_v:
7030 case NEON::BI__builtin_neon_vst4q_v:
7031 case NEON::BI__builtin_neon_vst2_lane_v:
7032 case NEON::BI__builtin_neon_vst2q_lane_v:
7033 case NEON::BI__builtin_neon_vst3_lane_v:
7034 case NEON::BI__builtin_neon_vst3q_lane_v:
7035 case NEON::BI__builtin_neon_vst4_lane_v:
7036 case NEON::BI__builtin_neon_vst4q_lane_v: {
7038 Ops.push_back(getAlignmentValue32(PtrOp0));
7041 case NEON::BI__builtin_neon_vsm3partw1q_v:
7042 case NEON::BI__builtin_neon_vsm3partw2q_v:
7043 case NEON::BI__builtin_neon_vsm3ss1q_v:
7044 case NEON::BI__builtin_neon_vsm4ekeyq_v:
7045 case NEON::BI__builtin_neon_vsm4eq_v: {
7049 case NEON::BI__builtin_neon_vsm3tt1aq_v:
7050 case NEON::BI__builtin_neon_vsm3tt1bq_v:
7051 case NEON::BI__builtin_neon_vsm3tt2aq_v:
7052 case NEON::BI__builtin_neon_vsm3tt2bq_v: {
7054 Ops[3] = Builder.CreateZExt(Ops[3],
Int64Ty);
7057 case NEON::BI__builtin_neon_vst1_x2_v:
7058 case NEON::BI__builtin_neon_vst1q_x2_v:
7059 case NEON::BI__builtin_neon_vst1_x3_v:
7060 case NEON::BI__builtin_neon_vst1q_x3_v:
7061 case NEON::BI__builtin_neon_vst1_x4_v:
7062 case NEON::BI__builtin_neon_vst1q_x4_v: {
7063 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getElementType());
7066 if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be ||
7067 Arch == llvm::Triple::aarch64_32) {
7068 llvm::Type *Tys[2] = { VTy, PTy };
7069 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
7072 llvm::Type *Tys[2] = { PTy, VTy };
7075 case NEON::BI__builtin_neon_vsubhn_v: {
7076 llvm::FixedVectorType *SrcTy =
7077 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
7080 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
7081 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
7082 Ops[0] = Builder.CreateSub(Ops[0], Ops[1],
"vsubhn");
7085 Constant *ShiftAmt =
7086 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
7087 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt,
"vsubhn");
7090 return Builder.CreateTrunc(Ops[0], VTy,
"vsubhn");
7092 case NEON::BI__builtin_neon_vtrn_v:
7093 case NEON::BI__builtin_neon_vtrnq_v: {
7094 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
7095 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7096 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7097 Value *SV =
nullptr;
7099 for (
unsigned vi = 0; vi != 2; ++vi) {
7101 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
7102 Indices.push_back(i+vi);
7103 Indices.push_back(i+e+vi);
7105 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
7106 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
7107 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
7111 case NEON::BI__builtin_neon_vtst_v:
7112 case NEON::BI__builtin_neon_vtstq_v: {
7113 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7114 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7115 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
7116 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
7117 ConstantAggregateZero::get(Ty));
7118 return Builder.CreateSExt(Ops[0], Ty,
"vtst");
7120 case NEON::BI__builtin_neon_vuzp_v:
7121 case NEON::BI__builtin_neon_vuzpq_v: {
7122 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
7123 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7124 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7125 Value *SV =
nullptr;
7127 for (
unsigned vi = 0; vi != 2; ++vi) {
7129 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
7130 Indices.push_back(2*i+vi);
7132 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
7133 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
7134 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
7138 case NEON::BI__builtin_neon_vxarq_v: {
7140 Ops[2] = Builder.CreateZExt(Ops[2],
Int64Ty);
7143 case NEON::BI__builtin_neon_vzip_v:
7144 case NEON::BI__builtin_neon_vzipq_v: {
7145 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
7146 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7147 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7148 Value *SV =
nullptr;
7150 for (
unsigned vi = 0; vi != 2; ++vi) {
7152 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
7153 Indices.push_back((i + vi*e) >> 1);
7154 Indices.push_back(((i + vi*e) >> 1)+e);
7156 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
7157 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
7158 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
7162 case NEON::BI__builtin_neon_vdot_v:
7163 case NEON::BI__builtin_neon_vdotq_v: {
7165 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
7166 llvm::Type *Tys[2] = { Ty, InputTy };
7167 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
7170 case NEON::BI__builtin_neon_vfmlal_low_v:
7171 case NEON::BI__builtin_neon_vfmlalq_low_v: {
7173 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
7174 llvm::Type *Tys[2] = { Ty, InputTy };
7177 case NEON::BI__builtin_neon_vfmlsl_low_v:
7178 case NEON::BI__builtin_neon_vfmlslq_low_v: {
7180 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
7181 llvm::Type *Tys[2] = { Ty, InputTy };
7184 case NEON::BI__builtin_neon_vfmlal_high_v:
7185 case NEON::BI__builtin_neon_vfmlalq_high_v: {
7187 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
7188 llvm::Type *Tys[2] = { Ty, InputTy };
7191 case NEON::BI__builtin_neon_vfmlsl_high_v:
7192 case NEON::BI__builtin_neon_vfmlslq_high_v: {
7194 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
7195 llvm::Type *Tys[2] = { Ty, InputTy };
7198 case NEON::BI__builtin_neon_vmmlaq_v: {
7200 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
7201 llvm::Type *Tys[2] = { Ty, InputTy };
7202 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
7205 case NEON::BI__builtin_neon_vusmmlaq_v: {
7207 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
7208 llvm::Type *Tys[2] = { Ty, InputTy };
7211 case NEON::BI__builtin_neon_vusdot_v:
7212 case NEON::BI__builtin_neon_vusdotq_v: {
7214 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
7215 llvm::Type *Tys[2] = { Ty, InputTy };
7218 case NEON::BI__builtin_neon_vbfdot_v:
7219 case NEON::BI__builtin_neon_vbfdotq_v: {
7220 llvm::Type *InputTy =
7221 llvm::FixedVectorType::get(
BFloatTy, Ty->getPrimitiveSizeInBits() / 16);
7222 llvm::Type *Tys[2] = { Ty, InputTy };
7225 case NEON::BI__builtin_neon___a32_vcvt_bf16_v: {
7226 llvm::Type *Tys[1] = { Ty };
7233 assert(Int &&
"Expected valid intrinsic number");
7242 return Builder.CreateBitCast(
Result, ResultType, NameHint);
7246 Value *Op, llvm::Type *Ty,
const CmpInst::Predicate Fp,
7247 const CmpInst::Predicate Ip,
const Twine &Name) {
7248 llvm::Type *OTy = Op->getType();
7254 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
7255 OTy = BI->getOperand(0)->getType();
7257 Op = Builder.CreateBitCast(Op, OTy);
7258 if (OTy->getScalarType()->isFloatingPointTy()) {
7259 if (Fp == CmpInst::FCMP_OEQ)
7260 Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
7262 Op = Builder.CreateFCmpS(Fp, Op, Constant::getNullValue(OTy));
7264 Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
7266 return Builder.CreateSExt(Op, Ty, Name);
7271 llvm::Type *ResTy,
unsigned IntID,
7275 TblOps.push_back(ExtOp);
7280 for (
unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
7281 Indices.push_back(2*i);
7282 Indices.push_back(2*i+1);
7285 int PairPos = 0, End = Ops.size() - 1;
7286 while (PairPos < End) {
7287 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
7288 Ops[PairPos+1], Indices,
7295 if (PairPos == End) {
7296 Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
7297 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
7298 ZeroTbl, Indices, Name));
7302 TblOps.push_back(IndexOp);
7308Value *CodeGenFunction::GetValueForARMHint(
unsigned BuiltinID) {
7310 switch (BuiltinID) {
7313 case clang::ARM::BI__builtin_arm_nop:
7316 case clang::ARM::BI__builtin_arm_yield:
7317 case clang::ARM::BI__yield:
7320 case clang::ARM::BI__builtin_arm_wfe:
7321 case clang::ARM::BI__wfe:
7324 case clang::ARM::BI__builtin_arm_wfi:
7325 case clang::ARM::BI__wfi:
7328 case clang::ARM::BI__builtin_arm_sev:
7329 case clang::ARM::BI__sev:
7332 case clang::ARM::BI__builtin_arm_sevl:
7333 case clang::ARM::BI__sevl:
7353 llvm::Type *RegisterType,
7354 llvm::Type *ValueType,
7356 StringRef SysReg =
"") {
7358 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64))
7359 &&
"Unsupported size for register.");
7365 if (SysReg.empty()) {
7370 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
7371 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
7372 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
7374 llvm::Type *Types[] = { RegisterType };
7376 bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
7377 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
7378 &&
"Can't fit 64-bit value in 32-bit register");
7380 if (AccessKind !=
Write) {
7383 AccessKind ==
VolatileRead ? llvm::Intrinsic::read_volatile_register
7384 : llvm::Intrinsic::read_register,
7386 llvm::Value *Call = Builder.CreateCall(F, Metadata);
7390 return Builder.CreateTrunc(Call, ValueType);
7392 if (ValueType->isPointerTy())
7394 return Builder.CreateIntToPtr(Call, ValueType);
7399 llvm::Function *F = CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
7403 ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
7404 return Builder.CreateCall(F, { Metadata, ArgValue });
7407 if (ValueType->isPointerTy()) {
7409 ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
7410 return Builder.CreateCall(F, { Metadata, ArgValue });
7413 return Builder.CreateCall(F, { Metadata, ArgValue });
7419 switch (BuiltinID) {
7421 case NEON::BI__builtin_neon_vget_lane_i8:
7422 case NEON::BI__builtin_neon_vget_lane_i16:
7423 case NEON::BI__builtin_neon_vget_lane_bf16:
7424 case NEON::BI__builtin_neon_vget_lane_i32:
7425 case NEON::BI__builtin_neon_vget_lane_i64:
7426 case NEON::BI__builtin_neon_vget_lane_f32:
7427 case NEON::BI__builtin_neon_vgetq_lane_i8:
7428 case NEON::BI__builtin_neon_vgetq_lane_i16:
7429 case NEON::BI__builtin_neon_vgetq_lane_bf16:
7430 case NEON::BI__builtin_neon_vgetq_lane_i32:
7431 case NEON::BI__builtin_neon_vgetq_lane_i64:
7432 case NEON::BI__builtin_neon_vgetq_lane_f32:
7433 case NEON::BI__builtin_neon_vduph_lane_bf16:
7434 case NEON::BI__builtin_neon_vduph_laneq_bf16:
7435 case NEON::BI__builtin_neon_vset_lane_i8:
7436 case NEON::BI__builtin_neon_vset_lane_i16:
7437 case NEON::BI__builtin_neon_vset_lane_bf16:
7438 case NEON::BI__builtin_neon_vset_lane_i32:
7439 case NEON::BI__builtin_neon_vset_lane_i64:
7440 case NEON::BI__builtin_neon_vset_lane_f32:
7441 case NEON::BI__builtin_neon_vsetq_lane_i8:
7442 case NEON::BI__builtin_neon_vsetq_lane_i16:
7443 case NEON::BI__builtin_neon_vsetq_lane_bf16:
7444 case NEON::BI__builtin_neon_vsetq_lane_i32:
7445 case NEON::BI__builtin_neon_vsetq_lane_i64:
7446 case NEON::BI__builtin_neon_vsetq_lane_f32:
7447 case NEON::BI__builtin_neon_vsha1h_u32:
7448 case NEON::BI__builtin_neon_vsha1cq_u32:
7449 case NEON::BI__builtin_neon_vsha1pq_u32:
7450 case NEON::BI__builtin_neon_vsha1mq_u32:
7451 case NEON::BI__builtin_neon_vcvth_bf16_f32:
7452 case clang::ARM::BI_MoveToCoprocessor:
7453 case clang::ARM::BI_MoveToCoprocessor2:
7462 llvm::Triple::ArchType Arch) {
7463 if (
auto Hint = GetValueForARMHint(BuiltinID))
7466 if (BuiltinID == clang::ARM::BI__emit) {
7468 llvm::FunctionType *FTy =
7469 llvm::FunctionType::get(
VoidTy,
false);
7473 llvm_unreachable(
"Sema will ensure that the parameter is constant");
7476 uint64_t ZExtValue =
Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
7478 llvm::InlineAsm *Emit =
7479 IsThumb ? InlineAsm::get(FTy,
".inst.n 0x" + utohexstr(ZExtValue),
"",
7481 : InlineAsm::get(FTy,
".inst 0x" + utohexstr(ZExtValue),
"",
7484 return Builder.CreateCall(Emit);
7487 if (BuiltinID == clang::ARM::BI__builtin_arm_dbg) {
7489 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::arm_dbg), Option);
7492 if (BuiltinID == clang::ARM::BI__builtin_arm_prefetch) {
7501 return Builder.CreateCall(F, {
Address, RW, Locality, IsData});
7504 if (BuiltinID == clang::ARM::BI__builtin_arm_rbit) {
7506 return Builder.CreateCall(
7507 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
7510 if (BuiltinID == clang::ARM::BI__builtin_arm_cls) {
7512 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::arm_cls), Arg,
"cls");
7514 if (BuiltinID == clang::ARM::BI__builtin_arm_cls64) {
7516 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::arm_cls64), Arg,
7520 if (BuiltinID == clang::ARM::BI__clear_cache) {
7521 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
7524 for (
unsigned i = 0; i < 2; i++)
7528 StringRef Name = FD->
getName();
7532 if (BuiltinID == clang::ARM::BI__builtin_arm_mcrr ||
7533 BuiltinID == clang::ARM::BI__builtin_arm_mcrr2) {
7536 switch (BuiltinID) {
7537 default: llvm_unreachable(
"unexpected builtin");
7538 case clang::ARM::BI__builtin_arm_mcrr:
7541 case clang::ARM::BI__builtin_arm_mcrr2:
7559 Value *Rt = Builder.CreateTruncOrBitCast(RtAndRt2,
Int32Ty);
7560 Value *Rt2 = Builder.CreateLShr(RtAndRt2, C1);
7561 Rt2 = Builder.CreateTruncOrBitCast(Rt2,
Int32Ty);
7563 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
7566 if (BuiltinID == clang::ARM::BI__builtin_arm_mrrc ||
7567 BuiltinID == clang::ARM::BI__builtin_arm_mrrc2) {
7570 switch (BuiltinID) {
7571 default: llvm_unreachable(
"unexpected builtin");
7572 case clang::ARM::BI__builtin_arm_mrrc:
7575 case clang::ARM::BI__builtin_arm_mrrc2:
7583 Value *RtAndRt2 = Builder.CreateCall(F, {Coproc, Opc1, CRm});
7588 Value *Rt = Builder.CreateExtractValue(RtAndRt2, 1);
7589 Value *Rt1 = Builder.CreateExtractValue(RtAndRt2, 0);
7590 Rt = Builder.CreateZExt(Rt,
Int64Ty);
7591 Rt1 = Builder.CreateZExt(Rt1,
Int64Ty);
7593 Value *ShiftCast = llvm::ConstantInt::get(
Int64Ty, 32);
7594 RtAndRt2 = Builder.CreateShl(Rt, ShiftCast,
"shl",
true);
7595 RtAndRt2 = Builder.CreateOr(RtAndRt2, Rt1);
7600 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrexd ||
7601 ((BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
7602 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) &&
7604 BuiltinID == clang::ARM::BI__ldrexd) {
7607 switch (BuiltinID) {
7608 default: llvm_unreachable(
"unexpected builtin");
7609 case clang::ARM::BI__builtin_arm_ldaex:
7612 case clang::ARM::BI__builtin_arm_ldrexd:
7613 case clang::ARM::BI__builtin_arm_ldrex:
7614 case clang::ARM::BI__ldrexd:
7620 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr,
Int8PtrTy),
7623 Value *Val0 = Builder.CreateExtractValue(Val, 1);
7624 Value *Val1 = Builder.CreateExtractValue(Val, 0);
7625 Val0 = Builder.CreateZExt(Val0,
Int64Ty);
7626 Val1 = Builder.CreateZExt(Val1,
Int64Ty);
7629 Val = Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
7630 Val = Builder.CreateOr(Val, Val1);
7634 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
7635 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) {
7642 llvm::Type *PtrTy =
IntTy->getPointerTo();
7643 LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy);
7646 BuiltinID == clang::ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex
7647 : Intrinsic::arm_ldrex,
7649 CallInst *Val = Builder.CreateCall(F, LoadAddr,
"ldrex");
7653 if (RealResTy->isPointerTy())
7654 return Builder.CreateIntToPtr(Val, RealResTy);
7656 llvm::Type *IntResTy = llvm::IntegerType::get(
7658 return Builder.CreateBitCast(Builder.CreateTruncOrBitCast(Val, IntResTy),
7663 if (BuiltinID == clang::ARM::BI__builtin_arm_strexd ||
7664 ((BuiltinID == clang::ARM::BI__builtin_arm_stlex ||
7665 BuiltinID == clang::ARM::BI__builtin_arm_strex) &&
7668 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlexd
7669 : Intrinsic::arm_strexd);
7674 Builder.CreateStore(Val, Tmp);
7676 Address LdPtr = Builder.CreateElementBitCast(Tmp, STy);
7677 Val = Builder.CreateLoad(LdPtr);
7679 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
7680 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
7682 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"strexd");
7685 if (BuiltinID == clang::ARM::BI__builtin_arm_strex ||
7686 BuiltinID == clang::ARM::BI__builtin_arm_stlex) {
7693 StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo());
7695 if (StoreVal->getType()->isPointerTy())
7696 StoreVal = Builder.CreatePtrToInt(StoreVal,
Int32Ty);
7698 llvm::Type *
IntTy = llvm::IntegerType::get(
7701 StoreVal = Builder.CreateBitCast(StoreVal,
IntTy);
7702 StoreVal = Builder.CreateZExtOrBitCast(StoreVal,
Int32Ty);
7706 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlex
7707 : Intrinsic::arm_strex,
7708 StoreAddr->getType());
7710 CallInst *CI = Builder.CreateCall(F, {StoreVal, StoreAddr},
"strex");
7712 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
7716 if (BuiltinID == clang::ARM::BI__builtin_arm_clrex) {
7718 return Builder.CreateCall(F);
7722 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
7723 switch (BuiltinID) {
7724 case clang::ARM::BI__builtin_arm_crc32b:
7725 CRCIntrinsicID = Intrinsic::arm_crc32b;
break;
7726 case clang::ARM::BI__builtin_arm_crc32cb:
7727 CRCIntrinsicID = Intrinsic::arm_crc32cb;
break;
7728 case clang::ARM::BI__builtin_arm_crc32h:
7729 CRCIntrinsicID = Intrinsic::arm_crc32h;
break;
7730 case clang::ARM::BI__builtin_arm_crc32ch:
7731 CRCIntrinsicID = Intrinsic::arm_crc32ch;
break;
7732 case clang::ARM::BI__builtin_arm_crc32w:
7733 case clang::ARM::BI__builtin_arm_crc32d:
7734 CRCIntrinsicID = Intrinsic::arm_crc32w;
break;
7735 case clang::ARM::BI__builtin_arm_crc32cw:
7736 case clang::ARM::BI__builtin_arm_crc32cd:
7737 CRCIntrinsicID = Intrinsic::arm_crc32cw;
break;
7740 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
7746 if (BuiltinID == clang::ARM::BI__builtin_arm_crc32d ||
7747 BuiltinID == clang::ARM::BI__builtin_arm_crc32cd) {
7749 Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1,
Int32Ty);
7750 Value *Arg1b = Builder.CreateLShr(Arg1, C1);
7751 Arg1b = Builder.CreateTruncOrBitCast(Arg1b,
Int32Ty);
7754 Value *Res = Builder.CreateCall(F, {Arg0, Arg1a});
7755 return Builder.CreateCall(F, {Res, Arg1b});
7757 Arg1 = Builder.CreateZExtOrBitCast(Arg1,
Int32Ty);
7760 return Builder.CreateCall(F, {Arg0, Arg1});
7764 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
7765 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
7766 BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
7767 BuiltinID == clang::ARM::BI__builtin_arm_wsr ||
7768 BuiltinID == clang::ARM::BI__builtin_arm_wsr64 ||
7769 BuiltinID == clang::ARM::BI__builtin_arm_wsrp) {
7772 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
7773 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
7774 BuiltinID == clang::ARM::BI__builtin_arm_rsrp)
7777 bool IsPointerBuiltin = BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
7778 BuiltinID == clang::ARM::BI__builtin_arm_wsrp;
7780 bool Is64Bit = BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
7781 BuiltinID == clang::ARM::BI__builtin_arm_wsr64;
7783 llvm::Type *ValueType;
7784 llvm::Type *RegisterType;
7785 if (IsPointerBuiltin) {
7788 }
else if (Is64Bit) {
7789 ValueType = RegisterType =
Int64Ty;
7791 ValueType = RegisterType =
Int32Ty;
7798 if (BuiltinID == ARM::BI__builtin_sponentry) {
7800 return Builder.CreateCall(F);
7817 unsigned ICEArguments = 0;
7822 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
7823 return Builder.getInt32(addr.getAlignment().getQuantity());
7830 unsigned NumArgs = E->
getNumArgs() - (HasExtraArg ? 1 : 0);
7831 for (
unsigned i = 0, e = NumArgs; i != e; i++) {
7833 switch (BuiltinID) {
7834 case NEON::BI__builtin_neon_vld1_v:
7835 case NEON::BI__builtin_neon_vld1q_v:
7836 case NEON::BI__builtin_neon_vld1q_lane_v:
7837 case NEON::BI__builtin_neon_vld1_lane_v:
7838 case NEON::BI__builtin_neon_vld1_dup_v:
7839 case NEON::BI__builtin_neon_vld1q_dup_v:
7840 case NEON::BI__builtin_neon_vst1_v:
7841 case NEON::BI__builtin_neon_vst1q_v:
7842 case NEON::BI__builtin_neon_vst1q_lane_v:
7843 case NEON::BI__builtin_neon_vst1_lane_v:
7844 case NEON::BI__builtin_neon_vst2_v:
7845 case NEON::BI__builtin_neon_vst2q_v:
7846 case NEON::BI__builtin_neon_vst2_lane_v:
7847 case NEON::BI__builtin_neon_vst2q_lane_v:
7848 case NEON::BI__builtin_neon_vst3_v:
7849 case NEON::BI__builtin_neon_vst3q_v:
7850 case NEON::BI__builtin_neon_vst3_lane_v:
7851 case NEON::BI__builtin_neon_vst3q_lane_v:
7852 case NEON::BI__builtin_neon_vst4_v:
7853 case NEON::BI__builtin_neon_vst4q_v:
7854 case NEON::BI__builtin_neon_vst4_lane_v:
7855 case NEON::BI__builtin_neon_vst4q_lane_v:
7864 switch (BuiltinID) {
7865 case NEON::BI__builtin_neon_vld2_v:
7866 case NEON::BI__builtin_neon_vld2q_v:
7867 case NEON::BI__builtin_neon_vld3_v:
7868 case NEON::BI__builtin_neon_vld3q_v:
7869 case NEON::BI__builtin_neon_vld4_v:
7870 case NEON::BI__builtin_neon_vld4q_v:
7871 case NEON::BI__builtin_neon_vld2_lane_v:
7872 case NEON::BI__builtin_neon_vld2q_lane_v:
7873 case NEON::BI__builtin_neon_vld3_lane_v:
7874 case NEON::BI__builtin_neon_vld3q_lane_v:
7875 case NEON::BI__builtin_neon_vld4_lane_v:
7876 case NEON::BI__builtin_neon_vld4q_lane_v:
7877 case NEON::BI__builtin_neon_vld2_dup_v:
7878 case NEON::BI__builtin_neon_vld2q_dup_v:
7879 case NEON::BI__builtin_neon_vld3_dup_v:
7880 case NEON::BI__builtin_neon_vld3q_dup_v:
7881 case NEON::BI__builtin_neon_vld4_dup_v:
7882 case NEON::BI__builtin_neon_vld4q_dup_v:
7891 if ((ICEArguments & (1 << i)) == 0) {
7896 Ops.push_back(llvm::ConstantInt::get(
7902 switch (BuiltinID) {
7905 case NEON::BI__builtin_neon_vget_lane_i8:
7906 case NEON::BI__builtin_neon_vget_lane_i16:
7907 case NEON::BI__builtin_neon_vget_lane_i32:
7908 case NEON::BI__builtin_neon_vget_lane_i64:
7909 case NEON::BI__builtin_neon_vget_lane_bf16:
7910 case NEON::BI__builtin_neon_vget_lane_f32:
7911 case NEON::BI__builtin_neon_vgetq_lane_i8:
7912 case NEON::BI__builtin_neon_vgetq_lane_i16:
7913 case NEON::BI__builtin_neon_vgetq_lane_i32:
7914 case NEON::BI__builtin_neon_vgetq_lane_i64:
7915 case NEON::BI__builtin_neon_vgetq_lane_bf16:
7916 case NEON::BI__builtin_neon_vgetq_lane_f32:
7917 case NEON::BI__builtin_neon_vduph_lane_bf16:
7918 case NEON::BI__builtin_neon_vduph_laneq_bf16:
7919 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
7921 case NEON::BI__builtin_neon_vrndns_f32: {
7923 llvm::Type *Tys[] = {Arg->getType()};
7925 return Builder.CreateCall(F, {Arg},
"vrndn"); }
7927 case NEON::BI__builtin_neon_vset_lane_i8:
7928 case NEON::BI__builtin_neon_vset_lane_i16:
7929 case NEON::BI__builtin_neon_vset_lane_i32:
7930 case NEON::BI__builtin_neon_vset_lane_i64:
7931 case NEON::BI__builtin_neon_vset_lane_bf16:
7932 case NEON::BI__builtin_neon_vset_lane_f32:
7933 case NEON::BI__builtin_neon_vsetq_lane_i8:
7934 case NEON::BI__builtin_neon_vsetq_lane_i16:
7935 case NEON::BI__builtin_neon_vsetq_lane_i32:
7936 case NEON::BI__builtin_neon_vsetq_lane_i64:
7937 case NEON::BI__builtin_neon_vsetq_lane_bf16:
7938 case NEON::BI__builtin_neon_vsetq_lane_f32:
7939 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
7941 case NEON::BI__builtin_neon_vsha1h_u32:
7944 case NEON::BI__builtin_neon_vsha1cq_u32:
7947 case NEON::BI__builtin_neon_vsha1pq_u32:
7950 case NEON::BI__builtin_neon_vsha1mq_u32:
7954 case NEON::BI__builtin_neon_vcvth_bf16_f32: {
7961 case clang::ARM::BI_MoveToCoprocessor:
7962 case clang::ARM::BI_MoveToCoprocessor2: {
7963 Function *F =
CGM.
getIntrinsic(BuiltinID == clang::ARM::BI_MoveToCoprocessor
7964 ? Intrinsic::arm_mcr
7965 : Intrinsic::arm_mcr2);
7966 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
7967 Ops[3], Ops[4], Ops[5]});
7972 assert(HasExtraArg);
7978 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f ||
7979 BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_d) {
7982 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f)
7988 bool usgn =
Result->getZExtValue() == 1;
7989 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
7993 return Builder.CreateCall(F, Ops,
"vcvtr");
7998 bool usgn =
Type.isUnsigned();
7999 bool rightShift =
false;
8001 llvm::FixedVectorType *VTy =
8004 llvm::Type *Ty = VTy;
8015 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
8016 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch);
8019 switch (BuiltinID) {
8020 default:
return nullptr;
8021 case NEON::BI__builtin_neon_vld1q_lane_v:
8024 if (VTy->getElementType()->isIntegerTy(64)) {
8026 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8028 Value *SV = llvm::ConstantVector::get(ConstantInt::get(
Int32Ty, 1-Lane));
8029 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
8031 Ty = llvm::FixedVectorType::get(VTy->getElementType(), 1);
8034 Value *Align = getAlignmentValue32(PtrOp0);
8035 Value *Ld = Builder.CreateCall(F, {Ops[0], Align});
8037 int Indices[] = {1 - Lane, Lane};
8038 return Builder.CreateShuffleVector(Ops[1], Ld, Indices,
"vld1q_lane");
8041 case NEON::BI__builtin_neon_vld1_lane_v: {
8042 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8043 PtrOp0 = Builder.CreateElementBitCast(PtrOp0, VTy->getElementType());
8044 Value *Ld = Builder.CreateLoad(PtrOp0);
8045 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2],
"vld1_lane");
8047 case NEON::BI__builtin_neon_vqrshrn_n_v:
8049 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
8052 case NEON::BI__builtin_neon_vqrshrun_n_v:
8054 Ops,
"vqrshrun_n", 1,
true);
8055 case NEON::BI__builtin_neon_vqshrn_n_v:
8056 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
8059 case NEON::BI__builtin_neon_vqshrun_n_v:
8061 Ops,
"vqshrun_n", 1,
true);
8062 case NEON::BI__builtin_neon_vrecpe_v:
8063 case NEON::BI__builtin_neon_vrecpeq_v:
8066 case NEON::BI__builtin_neon_vrshrn_n_v:
8068 Ops,
"vrshrn_n", 1,
true);
8069 case NEON::BI__builtin_neon_vrsra_n_v:
8070 case NEON::BI__builtin_neon_vrsraq_n_v:
8071 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8072 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8074 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
8075 Ops[1] = Builder.CreateCall(
CGM.
getIntrinsic(Int, Ty), {Ops[1], Ops[2]});
8076 return Builder.CreateAdd(Ops[0], Ops[1],
"vrsra_n");
8077 case NEON::BI__builtin_neon_vsri_n_v:
8078 case NEON::BI__builtin_neon_vsriq_n_v:
8081 case NEON::BI__builtin_neon_vsli_n_v:
8082 case NEON::BI__builtin_neon_vsliq_n_v:
8086 case NEON::BI__builtin_neon_vsra_n_v:
8087 case NEON::BI__builtin_neon_vsraq_n_v:
8088 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8090 return Builder.CreateAdd(Ops[0], Ops[1]);
8091 case NEON::BI__builtin_neon_vst1q_lane_v:
8094 if (VTy->getElementType()->isIntegerTy(64)) {
8095 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8096 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
8097 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
8098 Ops[2] = getAlignmentValue32(PtrOp0);
8099 llvm::Type *Tys[] = {
Int8PtrTy, Ops[1]->getType()};
8104 case NEON::BI__builtin_neon_vst1_lane_v: {
8105 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8106 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
8107 auto St = Builder.CreateStore(
8108 Ops[1], Builder.CreateElementBitCast(PtrOp0, Ops[1]->getType()));
8111 case NEON::BI__builtin_neon_vtbl1_v:
8114 case NEON::BI__builtin_neon_vtbl2_v:
8117 case NEON::BI__builtin_neon_vtbl3_v:
8120 case NEON::BI__builtin_neon_vtbl4_v:
8123 case NEON::BI__builtin_neon_vtbx1_v:
8126 case NEON::BI__builtin_neon_vtbx2_v:
8129 case NEON::BI__builtin_neon_vtbx3_v:
8132 case NEON::BI__builtin_neon_vtbx4_v:
8138template<
typename Integer>
8147 return Unsigned ? Builder.CreateZExt(
V, T) : Builder.CreateSExt(
V, T);
8159 ->getPrimitiveSizeInBits();
8160 if (Shift == LaneBits) {
8165 return llvm::Constant::getNullValue(
V->getType());
8169 return Unsigned ? Builder.CreateLShr(
V, Shift) : Builder.CreateAShr(
V, Shift);
8176 unsigned Elements = 128 /
V->getType()->getPrimitiveSizeInBits();
8177 return Builder.CreateVectorSplat(Elements,
V);
8183 llvm::Type *DestType) {
8196 V->getType()->getScalarSizeInBits() != DestType->getScalarSizeInBits()) {
8197 return Builder.CreateCall(
8199 {DestType, V->getType()}),
8202 return Builder.CreateBitCast(
V, DestType);
8210 unsigned InputElements =
8212 for (
unsigned i = 0; i < InputElements; i += 2)
8213 Indices.push_back(i + Odd);
8214 return Builder.CreateShuffleVector(
V, Indices);
8220 assert(V0->getType() == V1->getType() &&
"Can't zip different vector types");
8222 unsigned InputElements =
8224 for (
unsigned i = 0; i < InputElements; i++) {
8225 Indices.push_back(i);
8226 Indices.push_back(i + InputElements);
8228 return Builder.CreateShuffleVector(V0, V1, Indices);
8231template<
unsigned HighBit,
unsigned OtherBits>
8236 unsigned LaneBits = T->getPrimitiveSizeInBits();
8237 uint32_t
Value = HighBit << (LaneBits - 1);
8239 Value |= (1UL << (LaneBits - 1)) - 1;
8240 llvm::Value *Lane = llvm::ConstantInt::get(T,
Value);
8246 unsigned ReverseWidth) {
8250 unsigned LaneSize =
V->getType()->getScalarSizeInBits();
8251 unsigned Elements = 128 / LaneSize;
8252 unsigned Mask = ReverseWidth / LaneSize - 1;
8253 for (
unsigned i = 0; i < Elements; i++)
8254 Indices.push_back(i ^ Mask);
8255 return Builder.CreateShuffleVector(
V, Indices);
8261 llvm::Triple::ArchType Arch) {
8262 enum class CustomCodeGen { VLD24, VST24 } CustomCodeGenType;
8263 Intrinsic::ID IRIntr;
8264 unsigned NumVectors;
8267 switch (BuiltinID) {
8268 #include "clang/Basic/arm_mve_builtin_cg.inc"
8279 switch (CustomCodeGenType) {
8281 case CustomCodeGen::VLD24: {
8285 auto MvecCType = E->
getType();
8287 assert(MvecLType->isStructTy() &&
8288 "Return type for vld[24]q should be a struct");
8289 assert(MvecLType->getStructNumElements() == 1 &&
8290 "Return-type struct for vld[24]q should have one element");
8291 auto MvecLTypeInner = MvecLType->getStructElementType(0);
8292 assert(MvecLTypeInner->isArrayTy() &&
8293 "Return-type struct for vld[24]q should contain an array");
8294 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
8295 "Array member of return-type struct vld[24]q has wrong length");
8296 auto VecLType = MvecLTypeInner->getArrayElementType();
8298 Tys.push_back(VecLType);
8300 auto Addr = E->
getArg(0);
8305 Value *LoadResult = Builder.CreateCall(F, Ops);
8306 Value *MvecOut = UndefValue::get(MvecLType);
8307 for (
unsigned i = 0; i < NumVectors; ++i) {
8308 Value *Vec = Builder.CreateExtractValue(LoadResult, i);
8309 MvecOut = Builder.CreateInsertValue(MvecOut, Vec, {0, i});
8315 return Builder.CreateStore(MvecOut,
ReturnValue.getValue());
8318 case CustomCodeGen::VST24: {
8322 auto Addr = E->
getArg(0);
8328 assert(MvecLType->isStructTy() &&
"Data type for vst2q should be a struct");
8329 assert(MvecLType->getStructNumElements() == 1 &&
8330 "Data-type struct for vst2q should have one element");
8331 auto MvecLTypeInner = MvecLType->getStructElementType(0);
8332 assert(MvecLTypeInner->isArrayTy() &&
8333 "Data-type struct for vst2q should contain an array");
8334 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
8335 "Array member of return-type struct vld[24]q has wrong length");
8336 auto VecLType = MvecLTypeInner->getArrayElementType();
8338 Tys.push_back(VecLType);
8342 auto Mvec = Builder.CreateLoad(MvecSlot.
getAddress());
8343 for (
unsigned i = 0; i < NumVectors; i++)
8344 Ops.push_back(Builder.CreateExtractValue(Mvec, {0, i}));
8347 Value *ToReturn =
nullptr;
8348 for (
unsigned i = 0; i < NumVectors; i++) {
8349 Ops.push_back(llvm::ConstantInt::get(
Int32Ty, i));
8350 ToReturn = Builder.CreateCall(F, Ops);
8356 llvm_unreachable(
"unknown custom codegen type.");
8362 llvm::Triple::ArchType Arch) {
8363 switch (BuiltinID) {
8366#include "clang/Basic/arm_cde_builtin_cg.inc"
8373 llvm::Triple::ArchType Arch) {
8374 unsigned int Int = 0;
8375 const char *
s =
nullptr;
8377 switch (BuiltinID) {
8380 case NEON::BI__builtin_neon_vtbl1_v:
8381 case NEON::BI__builtin_neon_vqtbl1_v:
8382 case NEON::BI__builtin_neon_vqtbl1q_v:
8383 case NEON::BI__builtin_neon_vtbl2_v:
8384 case NEON::BI__builtin_neon_vqtbl2_v:
8385 case NEON::BI__builtin_neon_vqtbl2q_v:
8386 case NEON::BI__builtin_neon_vtbl3_v:
8387 case NEON::BI__builtin_neon_vqtbl3_v:
8388 case NEON::BI__builtin_neon_vqtbl3q_v:
8389 case NEON::BI__builtin_neon_vtbl4_v:
8390 case NEON::BI__builtin_neon_vqtbl4_v:
8391 case NEON::BI__builtin_neon_vqtbl4q_v:
8393 case NEON::BI__builtin_neon_vtbx1_v:
8394 case NEON::BI__builtin_neon_vqtbx1_v:
8395 case NEON::BI__builtin_neon_vqtbx1q_v:
8396 case NEON::BI__builtin_neon_vtbx2_v:
8397 case NEON::BI__builtin_neon_vqtbx2_v:
8398 case NEON::BI__builtin_neon_vqtbx2q_v:
8399 case NEON::BI__builtin_neon_vtbx3_v:
8400 case NEON::BI__builtin_neon_vqtbx3_v:
8401 case NEON::BI__builtin_neon_vqtbx3q_v:
8402 case NEON::BI__builtin_neon_vtbx4_v:
8403 case NEON::BI__builtin_neon_vqtbx4_v:
8404 case NEON::BI__builtin_neon_vqtbx4q_v:
8426 switch (BuiltinID) {
8427 case NEON::BI__builtin_neon_vtbl1_v: {
8429 Ops[1], Ty, Intrinsic::aarch64_neon_tbl1,
8432 case NEON::BI__builtin_neon_vtbl2_v: {
8434 Ops[2], Ty, Intrinsic::aarch64_neon_tbl1,
8437 case NEON::BI__builtin_neon_vtbl3_v: {
8439 Ops[3], Ty, Intrinsic::aarch64_neon_tbl2,
8442 case NEON::BI__builtin_neon_vtbl4_v: {
8444 Ops[4], Ty, Intrinsic::aarch64_neon_tbl2,
8447 case NEON::BI__builtin_neon_vtbx1_v: {
8450 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
8452 llvm::Constant *EightV = ConstantInt::get(Ty, 8);
8453 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
8454 CmpRes = Builder.CreateSExt(CmpRes, Ty);
8456 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
8457 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
8458 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
8460 case NEON::BI__builtin_neon_vtbx2_v: {
8462 Ops[3], Ty, Intrinsic::aarch64_neon_tbx1,
8465 case NEON::BI__builtin_neon_vtbx3_v: {
8468 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
8470 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
8471 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
8473 CmpRes = Builder.CreateSExt(CmpRes, Ty);
8475 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
8476 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
8477 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
8479 case NEON::BI__builtin_neon_vtbx4_v: {
8481 Ops[5], Ty, Intrinsic::aarch64_neon_tbx2,
8484 case NEON::BI__builtin_neon_vqtbl1_v:
8485 case NEON::BI__builtin_neon_vqtbl1q_v:
8486 Int = Intrinsic::aarch64_neon_tbl1;
s =
"vtbl1";
break;
8487 case NEON::BI__builtin_neon_vqtbl2_v:
8488 case NEON::BI__builtin_neon_vqtbl2q_v: {
8489 Int = Intrinsic::aarch64_neon_tbl2;
s =
"vtbl2";
break;
8490 case NEON::BI__builtin_neon_vqtbl3_v:
8491 case NEON::BI__builtin_neon_vqtbl3q_v:
8492 Int = Intrinsic::aarch64_neon_tbl3;
s =
"vtbl3";
break;
8493 case NEON::BI__builtin_neon_vqtbl4_v:
8494 case NEON::BI__builtin_neon_vqtbl4q_v:
8495 Int = Intrinsic::aarch64_neon_tbl4;
s =
"vtbl4";
break;
8496 case NEON::BI__builtin_neon_vqtbx1_v:
8497 case NEON::BI__builtin_neon_vqtbx1q_v:
8498 Int = Intrinsic::aarch64_neon_tbx1;
s =
"vtbx1";
break;
8499 case NEON::BI__builtin_neon_vqtbx2_v:
8500 case NEON::BI__builtin_neon_vqtbx2q_v:
8501 Int = Intrinsic::aarch64_neon_tbx2;
s =
"vtbx2";
break;
8502 case NEON::BI__builtin_neon_vqtbx3_v:
8503 case NEON::BI__builtin_neon_vqtbx3q_v:
8504 Int = Intrinsic::aarch64_neon_tbx3;
s =
"vtbx3";
break;
8505 case NEON::BI__builtin_neon_vqtbx4_v:
8506 case NEON::BI__builtin_neon_vqtbx4q_v:
8507 Int = Intrinsic::aarch64_neon_tbx4;
s =
"vtbx4";
break;
8519 auto *VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
8520 Op = Builder.CreateBitCast(Op,
Int16Ty);
8521 Value *
V = UndefValue::get(VTy);
8522 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
8523 Op = Builder.CreateInsertElement(
V, Op, CI);
8532 case SVETypeFlags::MemEltTyDefault:
8534 case SVETypeFlags::MemEltTyInt8:
8535 return Builder.getInt8Ty();
8536 case SVETypeFlags::MemEltTyInt16:
8537 return Builder.getInt16Ty();
8538 case SVETypeFlags::MemEltTyInt32:
8539 return Builder.getInt32Ty();
8540 case SVETypeFlags::MemEltTyInt64:
8541 return Builder.getInt64Ty();
8543 llvm_unreachable(
"Unknown MemEltType");
8549 llvm_unreachable(
"Invalid SVETypeFlag!");
8551 case SVETypeFlags::EltTyInt8:
8552 return Builder.getInt8Ty();
8553 case SVETypeFlags::EltTyInt16:
8554 return Builder.getInt16Ty();
8555 case SVETypeFlags::EltTyInt32:
8556 return Builder.getInt32Ty();
8557 case SVETypeFlags::EltTyInt64:
8558 return Builder.getInt64Ty();
8560 case SVETypeFlags::EltTyFloat16:
8561 return Builder.getHalfTy();
8562 case SVETypeFlags::EltTyFloat32:
8563 return Builder.getFloatTy();
8564 case SVETypeFlags::EltTyFloat64:
8565 return Builder.getDoubleTy();
8567 case SVETypeFlags::EltTyBFloat16:
8568 return Builder.getBFloatTy();
8570 case SVETypeFlags::EltTyBool8:
8571 case SVETypeFlags::EltTyBool16:
8572 case SVETypeFlags::EltTyBool32:
8573 case SVETypeFlags::EltTyBool64:
8574 return Builder.getInt1Ty();
8580llvm::ScalableVectorType *
8583 default: llvm_unreachable(
"Unhandled SVETypeFlag!");
8585 case SVETypeFlags::EltTyInt8:
8586 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
8587 case SVETypeFlags::EltTyInt16:
8588 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
8589 case SVETypeFlags::EltTyInt32:
8590 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
8591 case SVETypeFlags::EltTyInt64:
8592 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
8594 case SVETypeFlags::EltTyBFloat16:
8595 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
8596 case SVETypeFlags::EltTyFloat16:
8597 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
8598 case SVETypeFlags::EltTyFloat32:
8599 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
8600 case SVETypeFlags::EltTyFloat64:
8601 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
8603 case SVETypeFlags::EltTyBool8:
8604 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
8605 case SVETypeFlags::EltTyBool16:
8606 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
8607 case SVETypeFlags::EltTyBool32:
8608 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
8609 case SVETypeFlags::EltTyBool64:
8610 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
8615llvm::ScalableVectorType *
8619 llvm_unreachable(
"Invalid SVETypeFlag!");
8621 case SVETypeFlags::EltTyInt8:
8622 return llvm::ScalableVectorType::get(Builder.getInt8Ty(), 16);
8623 case SVETypeFlags::EltTyInt16:
8624 return llvm::ScalableVectorType::get(Builder.getInt16Ty(), 8);
8625 case SVETypeFlags::EltTyInt32:
8626 return llvm::ScalableVectorType::get(Builder.getInt32Ty(), 4);
8627 case SVETypeFlags::EltTyInt64:
8628 return llvm::ScalableVectorType::get(Builder.getInt64Ty(), 2);
8630 case SVETypeFlags::EltTyFloat16:
8631 return llvm::ScalableVectorType::get(Builder.getHalfTy(), 8);
8632 case SVETypeFlags::EltTyBFloat16:
8633 return llvm::ScalableVectorType::get(Builder.getBFloatTy(), 8);
8634 case SVETypeFlags::EltTyFloat32:
8635 return llvm::ScalableVectorType::get(Builder.getFloatTy(), 4);
8636 case SVETypeFlags::EltTyFloat64:
8637 return llvm::ScalableVectorType::get(Builder.getDoubleTy(), 2);
8639 case SVETypeFlags::EltTyBool8:
8640 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
8641 case SVETypeFlags::EltTyBool16:
8642 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
8643 case SVETypeFlags::EltTyBool32:
8644 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
8645 case SVETypeFlags::EltTyBool64:
8646 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
8654 return Builder.CreateCall(Ptrue, {Builder.getInt32( 31)});
8661 return llvm::ScalableVectorType::get(EltTy, NumElts);
8667 llvm::ScalableVectorType *VTy) {
8668 auto *RTy = llvm::VectorType::get(IntegerType::get(
getLLVMContext(), 1), VTy);
8669 if (Pred->getType() == RTy)
8673 llvm::Type *IntrinsicTy;
8674 switch (VTy->getMinNumElements()) {
8676 llvm_unreachable(
"unsupported element count!");
8680 IntID = Intrinsic::aarch64_sve_convert_from_svbool;
8684 IntID = Intrinsic::aarch64_sve_convert_to_svbool;
8685 IntrinsicTy = Pred->getType();
8690 Value *
C = Builder.CreateCall(F, Pred);
8691 assert(
C->getType() == RTy &&
"Unexpected return type!");
8699 auto *OverloadedTy =
8709 Function *F =
nullptr;
8710 if (Ops[1]->
getType()->isVectorTy())
8725 if (Ops.size() == 2) {
8726 assert(Ops[1]->
getType()->isVectorTy() &&
"Scalar base requires an offset");
8727 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
8732 if (!TypeFlags.
isByteIndexed() && Ops[1]->getType()->isVectorTy()) {
8733 unsigned BytesPerElt =
8734 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
8736 Ops[2] = Builder.CreateMul(Ops[2], Scale);
8739 Value *
Call = Builder.CreateCall(F, Ops);
8743 return TypeFlags.
isZExtReturn() ? Builder.CreateZExt(Call, ResultTy)
8744 : Builder.CreateSExt(Call, ResultTy);
8751 auto *OverloadedTy =
8756 Ops.insert(Ops.begin(), Ops.pop_back_val());
8758 Function *F =
nullptr;
8759 if (Ops[2]->
getType()->isVectorTy())
8774 if (Ops.size() == 3) {
8775 assert(Ops[1]->
getType()->isVectorTy() &&
"Scalar base requires an offset");
8776 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
8781 Ops[0] = Builder.CreateTrunc(Ops[0], OverloadedTy);
8792 if (!TypeFlags.
isByteIndexed() && Ops[2]->getType()->isVectorTy()) {
8793 unsigned BytesPerElt =
8794 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
8796 Ops[3] = Builder.CreateMul(Ops[3], Scale);
8799 return Builder.CreateCall(F, Ops);
8807 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Ops[1]->
getType());
8815 if (Ops[1]->
getType()->isVectorTy()) {
8816 if (Ops.size() == 3) {
8818 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
8821 std::swap(Ops[2], Ops[3]);
8825 unsigned BytesPerElt = MemEltTy->getPrimitiveSizeInBits() / 8;
8827 Ops[2] = Builder.CreateMul(Ops[2], Scale);
8832 return Builder.CreateCall(F, Ops);
8838 llvm::ScalableVectorType *VTy =
getSVEType(TypeFlags);
8839 auto VecPtrTy = llvm::PointerType::getUnqual(VTy);
8840 auto EltPtrTy = llvm::PointerType::getUnqual(VTy->getElementType());
8844 case Intrinsic::aarch64_sve_ld2:
8847 case Intrinsic::aarch64_sve_ld3:
8850 case Intrinsic::aarch64_sve_ld4:
8854 llvm_unreachable(
"unknown intrinsic!");
8856 auto RetTy = llvm::VectorType::get(VTy->getElementType(),
8857 VTy->getElementCount() *
N);
8860 Value *BasePtr= Builder.CreateBitCast(Ops[1], VecPtrTy);
8861 Value *
Offset = Ops.size() > 2 ? Ops[2] : Builder.getInt32(0);
8862 BasePtr = Builder.CreateGEP(VTy, BasePtr,
Offset);
8863 BasePtr = Builder.CreateBitCast(BasePtr, EltPtrTy);
8866 return Builder.CreateCall(F, { Predicate, BasePtr });
8872 llvm::ScalableVectorType *VTy =
getSVEType(TypeFlags);
8873 auto VecPtrTy = llvm::PointerType::getUnqual(VTy);
8874 auto EltPtrTy = llvm::PointerType::getUnqual(VTy->getElementType());
8878 case Intrinsic::aarch64_sve_st2:
8881 case Intrinsic::aarch64_sve_st3:
8884 case Intrinsic::aarch64_sve_st4:
8888 llvm_unreachable(
"unknown intrinsic!");
8891 llvm::VectorType::get(VTy->getElementType(), VTy->getElementCount() *
N);
8894 Value *BasePtr = Builder.CreateBitCast(Ops[1], VecPtrTy);
8895 Value *
Offset = Ops.size() > 3 ? Ops[2] : Builder.getInt32(0);
8896 Value *Val = Ops.back();
8897 BasePtr = Builder.CreateGEP(VTy, BasePtr,
Offset);
8898 BasePtr = Builder.CreateBitCast(BasePtr, EltPtrTy);
8905 for (
unsigned I = 0; I <
N; ++I)
8906 Operands.push_back(Builder.CreateCall(FExtr, {Val, Builder.getInt32(I)}));
8907 Operands.append({Predicate, BasePtr});
8910 return Builder.CreateCall(F, Operands);
8918 unsigned BuiltinID) {
8927 Value *
Call = Builder.CreateCall(F, {Ops[0], Ops[1]});
8930 llvm::ScalableVectorType *Ty =
getSVEType(TypeFlags);
8936 llvm::Type *OverloadedTy =
getSVEType(TypeFlags);
8938 return Builder.CreateCall(F, {Ops[0], Builder.getInt32(0)});
8943 unsigned BuiltinID) {
8946 auto *MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
8949 Value *BasePtr = Ops[1];
8952 if (Ops.size() > 3) {
8953 BasePtr = Builder.CreateBitCast(BasePtr, MemoryTy->getPointerTo());
8954 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]);
8958 BasePtr = Builder.CreateBitCast(BasePtr, llvm::PointerType::getUnqual(
Int8Ty));
8959 Value *PrfOp = Ops.back();
8962 return Builder.CreateCall(F, {Predicate, BasePtr, PrfOp});
8966 llvm::Type *ReturnTy,
8969 bool IsZExtReturn) {
8977 auto MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
8980 Value *BasePtr = Builder.CreateBitCast(Ops[1], MemoryTy->getPointerTo());
8981 Value *
Offset = Ops.size() > 2 ? Ops[2] : Builder.getInt32(0);
8982 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr,
Offset);
8984 BasePtr = Builder.CreateBitCast(BasePtr, MemEltTy->getPointerTo());
8991 return IsZExtReturn ? Builder.CreateZExt(Load, VectorTy)
8992 : Builder.CreateSExt(Load, VectorTy);
8997 unsigned BuiltinID) {
9005 auto MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
9008 Value *BasePtr = Builder.CreateBitCast(Ops[1], MemoryTy->getPointerTo());
9009 Value *
Offset = Ops.size() == 4 ? Ops[2] : Builder.getInt32(0);
9010 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr,
Offset);
9013 llvm::Value *Val = Builder.CreateTrunc(Ops.back(), MemoryTy);
9015 BasePtr = Builder.CreateBitCast(BasePtr, MemEltTy->getPointerTo());
9028 return Builder.CreateCall(F, Scalar);
9042 return Builder.CreateBitCast(Val, Ty);
9047 auto *SplatZero = Constant::getNullValue(Ty);
9048 Ops.insert(Ops.begin(), SplatZero);
9053 auto *SplatUndef = UndefValue::get(Ty);
9054 Ops.insert(Ops.begin(), SplatUndef);
9059 llvm::Type *ResultType,
9064 llvm::Type *DefaultType =
getSVEType(TypeFlags);
9067 return {DefaultType, Ops[1]->getType()};
9073 return {Ops[0]->getType(), Ops.back()->getType()};
9076 return {ResultType, Ops[0]->getType()};
9079 return {DefaultType};
9085 unsigned ICEArguments = 0;
9091 if (BuiltinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
9092 BuiltinID <= SVE::BI__builtin_sve_reinterpret_f64_f64) {
9098 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
9099 if ((ICEArguments & (1 << i)) == 0)
9106 assert(
Result &&
"Expected argument to be a constant");
9137 return UndefValue::get(Ty);
9138 else if (Builtin->LLVMIntrinsic != 0) {
9139 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZeroExp)
9142 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeAnyExp)
9148 Ops.push_back(Builder.getInt32( 31));
9150 Ops.insert(&Ops[1], Builder.getInt32( 31));
9153 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
9154 if (
auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->
getType()))
9155 if (PredTy->getElementType()->isIntegerTy(1))
9165 std::swap(Ops[1], Ops[2]);
9168 std::swap(Ops[1], Ops[2]);
9171 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZero) {
9172 llvm::Type *OpndTy = Ops[1]->getType();
9173 auto *SplatZero = Constant::getNullValue(OpndTy);
9175 Ops[1] = Builder.CreateCall(Sel, {Ops[0], Ops[1], SplatZero});
9180 Value *
Call = Builder.CreateCall(F, Ops);
9183 if (
auto PredTy = dyn_cast<llvm::VectorType>(
Call->getType()))
9184 if (PredTy->getScalarType()->isIntegerTy(1))
9190 switch (BuiltinID) {
9194 case SVE::BI__builtin_sve_svmov_b_z: {
9197 llvm::Type* OverloadedTy =
getSVEType(TypeFlags);
9198 Function *F =
CGM.
getIntrinsic(Intrinsic::aarch64_sve_and_z, OverloadedTy);
9199 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[1]});
9202 case SVE::BI__builtin_sve_svnot_b_z: {
9205 llvm::Type* OverloadedTy =
getSVEType(TypeFlags);
9206 Function *F =
CGM.
getIntrinsic(Intrinsic::aarch64_sve_eor_z, OverloadedTy);
9207 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[0]});
9210 case SVE::BI__builtin_sve_svmovlb_u16:
9211 case SVE::BI__builtin_sve_svmovlb_u32:
9212 case SVE::BI__builtin_sve_svmovlb_u64:
9213 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllb);
9215 case SVE::BI__builtin_sve_svmovlb_s16:
9216 case SVE::BI__builtin_sve_svmovlb_s32:
9217 case SVE::BI__builtin_sve_svmovlb_s64:
9218 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllb);
9220 case SVE::BI__builtin_sve_svmovlt_u16:
9221 case SVE::BI__builtin_sve_svmovlt_u32:
9222 case SVE::BI__builtin_sve_svmovlt_u64:
9223 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllt);
9225 case SVE::BI__builtin_sve_svmovlt_s16:
9226 case SVE::BI__builtin_sve_svmovlt_s32:
9227 case SVE::BI__builtin_sve_svmovlt_s64:
9228 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllt);
9230 case SVE::BI__builtin_sve_svpmullt_u16:
9231 case SVE::BI__builtin_sve_svpmullt_u64:
9232 case SVE::BI__builtin_sve_svpmullt_n_u16:
9233 case SVE::BI__builtin_sve_svpmullt_n_u64:
9234 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullt_pair);
9236 case SVE::BI__builtin_sve_svpmullb_u16:
9237 case SVE::BI__builtin_sve_svpmullb_u64:
9238 case SVE::BI__builtin_sve_svpmullb_n_u16:
9239 case SVE::BI__builtin_sve_svpmullb_n_u64:
9240 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullb_pair);
9242 case SVE::BI__builtin_sve_svdup_n_b8:
9243 case SVE::BI__builtin_sve_svdup_n_b16:
9244 case SVE::BI__builtin_sve_svdup_n_b32:
9245 case SVE::BI__builtin_sve_svdup_n_b64: {
9247 Builder.CreateICmpNE(Ops[0], Constant::getNullValue(Ops[0]->
getType()));
9248 llvm::ScalableVectorType *OverloadedTy =
getSVEType(TypeFlags);
9253 case SVE::BI__builtin_sve_svdupq_n_b8:
9254 case SVE::BI__builtin_sve_svdupq_n_b16:
9255 case SVE::BI__builtin_sve_svdupq_n_b32:
9256 case SVE::BI__builtin_sve_svdupq_n_b64:
9257 case SVE::BI__builtin_sve_svdupq_n_u8:
9258 case SVE::BI__builtin_sve_svdupq_n_s8:
9259 case SVE::BI__builtin_sve_svdupq_n_u64:
9260 case SVE::BI__builtin_sve_svdupq_n_f64:
9261 case SVE::BI__builtin_sve_svdupq_n_s64:
9262 case SVE::BI__builtin_sve_svdupq_n_u16:
9263 case SVE::BI__builtin_sve_svdupq_n_f16:
9264 case SVE::BI__builtin_sve_svdupq_n_bf16:
9265 case SVE::BI__builtin_sve_svdupq_n_s16:
9266 case SVE::BI__builtin_sve_svdupq_n_u32:
9267 case SVE::BI__builtin_sve_svdupq_n_f32:
9268 case SVE::BI__builtin_sve_svdupq_n_s32: {
9271 unsigned NumOpnds = Ops.size();
9279 llvm::Type *EltTy = Ops[0]->getType();
9284 for (
unsigned I = 0; I < NumOpnds; ++I)
9285 VecOps.push_back(Builder.CreateZExt(Ops[I], EltTy));
9292 Value *InsertSubVec = Builder.CreateInsertVector(
9293 OverloadedTy, UndefValue::get(OverloadedTy), Vec, Builder.getInt64(0));
9298 Builder.CreateCall(F, {InsertSubVec, Builder.getInt64(0)});
9305 : Intrinsic::aarch64_sve_cmpne_wide,
9308 F, {Pred, DupQLane,
EmitSVEDupX(Builder.getInt64(0))});
9312 case SVE::BI__builtin_sve_svpfalse_b:
9313 return ConstantInt::getFalse(Ty);
9315 case SVE::BI__builtin_sve_svlen_bf16:
9316 case SVE::BI__builtin_sve_svlen_f16:
9317 case SVE::BI__builtin_sve_svlen_f32:
9318 case SVE::BI__builtin_sve_svlen_f64:
9319 case SVE::BI__builtin_sve_svlen_s8:
9320 case SVE::BI__builtin_sve_svlen_s16:
9321 case SVE::BI__builtin_sve_svlen_s32:
9322 case SVE::BI__builtin_sve_svlen_s64:
9323 case SVE::BI__builtin_sve_svlen_u8:
9324 case SVE::BI__builtin_sve_svlen_u16:
9325 case SVE::BI__builtin_sve_svlen_u32:
9326 case SVE::BI__builtin_sve_svlen_u64: {
9330 llvm::ConstantInt::get(Ty, VTy->getElementCount().getKnownMinValue());
9333 return Builder.CreateMul(NumEls, Builder.CreateCall(F));
9336 case SVE::BI__builtin_sve_svtbl2_u8:
9337 case SVE::BI__builtin_sve_svtbl2_s8:
9338 case SVE::BI__builtin_sve_svtbl2_u16:
9339 case SVE::BI__builtin_sve_svtbl2_s16:
9340 case SVE::BI__builtin_sve_svtbl2_u32:
9341 case SVE::BI__builtin_sve_svtbl2_s32:
9342 case SVE::BI__builtin_sve_svtbl2_u64:
9343 case SVE::BI__builtin_sve_svtbl2_s64:
9344 case SVE::BI__builtin_sve_svtbl2_f16:
9345 case SVE::BI__builtin_sve_svtbl2_bf16:
9346 case SVE::BI__builtin_sve_svtbl2_f32:
9347 case SVE::BI__builtin_sve_svtbl2_f64: {
9350 auto TupleTy = llvm::VectorType::getDoubleElementsVectorType(VTy);
9353 Value *V0 = Builder.CreateCall(FExtr, {Ops[0], Builder.getInt32(0)});
9354 Value *V1 = Builder.CreateCall(FExtr, {Ops[0], Builder.getInt32(1)});
9356 return Builder.CreateCall(F, {V0, V1, Ops[1]});
9359 case SVE::BI__builtin_sve_svset_neonq_s8:
9360 case SVE::BI__builtin_sve_svset_neonq_s16:
9361 case SVE::BI__builtin_sve_svset_neonq_s32:
9362 case SVE::BI__builtin_sve_svset_neonq_s64:
9363 case SVE::BI__builtin_sve_svset_neonq_u8:
9364 case SVE::BI__builtin_sve_svset_neonq_u16:
9365 case SVE::BI__builtin_sve_svset_neonq_u32:
9366 case SVE::BI__builtin_sve_svset_neonq_u64:
9367 case SVE::BI__builtin_sve_svset_neonq_f16:
9368 case SVE::BI__builtin_sve_svset_neonq_f32:
9369 case SVE::BI__builtin_sve_svset_neonq_f64:
9370 case SVE::BI__builtin_sve_svset_neonq_bf16: {
9371 return Builder.CreateInsertVector(Ty, Ops[0], Ops[1], Builder.getInt64(0));
9374 case SVE::BI__builtin_sve_svget_neonq_s8:
9375 case SVE::BI__builtin_sve_svget_neonq_s16:
9376 case SVE::BI__builtin_sve_svget_neonq_s32:
9377 case SVE::BI__builtin_sve_svget_neonq_s64:
9378 case SVE::BI__builtin_sve_svget_neonq_u8:
9379 case SVE::BI__builtin_sve_svget_neonq_u16:
9380 case SVE::BI__builtin_sve_svget_neonq_u32:
9381 case SVE::BI__builtin_sve_svget_neonq_u64:
9382 case SVE::BI__builtin_sve_svget_neonq_f16:
9383 case SVE::BI__builtin_sve_svget_neonq_f32:
9384 case SVE::BI__builtin_sve_svget_neonq_f64:
9385 case SVE::BI__builtin_sve_svget_neonq_bf16: {
9386 return Builder.CreateExtractVector(Ty, Ops[0], Builder.getInt64(0));
9389 case SVE::BI__builtin_sve_svdup_neonq_s8:
9390 case SVE::BI__builtin_sve_svdup_neonq_s16:
9391 case SVE::BI__builtin_sve_svdup_neonq_s32:
9392 case SVE::BI__builtin_sve_svdup_neonq_s64:
9393 case SVE::BI__builtin_sve_svdup_neonq_u8:
9394 case SVE::BI__builtin_sve_svdup_neonq_u16:
9395 case SVE::BI__builtin_sve_svdup_neonq_u32:
9396 case SVE::BI__builtin_sve_svdup_neonq_u64:
9397 case SVE::BI__builtin_sve_svdup_neonq_f16:
9398 case SVE::BI__builtin_sve_svdup_neonq_f32:
9399 case SVE::BI__builtin_sve_svdup_neonq_f64:
9400 case SVE::BI__builtin_sve_svdup_neonq_bf16: {
9401 Value *
Insert = Builder.CreateInsertVector(Ty, UndefValue::get(Ty), Ops[0],
9402 Builder.getInt64(0));
9403 return Builder.CreateIntrinsic(Intrinsic::aarch64_sve_dupq_lane, {Ty},
9404 {
Insert, Builder.getInt64(0)});
9414 llvm::Triple::ArchType Arch) {
9419 unsigned HintID =
static_cast<unsigned>(-1);
9420 switch (BuiltinID) {
9422 case clang::AArch64::BI__builtin_arm_nop:
9425 case clang::AArch64::BI__builtin_arm_yield:
9426 case clang::AArch64::BI__yield:
9429 case clang::AArch64::BI__builtin_arm_wfe:
9430 case clang::AArch64::BI__wfe:
9433 case clang::AArch64::BI__builtin_arm_wfi:
9434 case clang::AArch64::BI__wfi:
9437 case clang::AArch64::BI__builtin_arm_sev:
9438 case clang::AArch64::BI__sev:
9441 case clang::AArch64::BI__builtin_arm_sevl:
9442 case clang::AArch64::BI__sevl:
9447 if (HintID !=
static_cast<unsigned>(-1)) {
9449 return Builder.CreateCall(F, llvm::ConstantInt::get(
Int32Ty, HintID));
9452 if (BuiltinID == clang::AArch64::BI__builtin_arm_prefetch) {
9459 Value *Locality =
nullptr;
9460 if (cast<llvm::ConstantInt>(RetentionPolicy)->isZero()) {
9462 Locality = llvm::ConstantInt::get(
Int32Ty,
9463 -cast<llvm::ConstantInt>(CacheLevel)->
getValue() + 3);
9466 Locality = llvm::ConstantInt::get(
Int32Ty, 0);
9472 return Builder.CreateCall(F, {
Address, RW, Locality, IsData});
9475 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
9477 "rbit of unusual size!");
9479 return Builder.CreateCall(
9480 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
9482 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit64) {
9484 "rbit of unusual size!");
9486 return Builder.CreateCall(
9487 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
9490 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls) {
9492 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::aarch64_cls), Arg,
9495 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls64) {
9497 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::aarch64_cls64), Arg,
9501 if (BuiltinID == clang::AArch64::BI__builtin_arm_frint32zf ||
9502 BuiltinID == clang::AArch64::BI__builtin_arm_frint32z) {
9504 llvm::Type *Ty = Arg->getType();
9505 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::aarch64_frint32z, Ty),
9509 if (BuiltinID == clang::AArch64::BI__builtin_arm_frint64zf ||
9510 BuiltinID == clang::AArch64::BI__builtin_arm_frint64z) {
9512 llvm::Type *Ty = Arg->getType();
9513 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::aarch64_frint64z, Ty),
9517 if (BuiltinID == clang::AArch64::BI__builtin_arm_frint32xf ||
9518 BuiltinID == clang::AArch64::BI__builtin_arm_frint32x) {
9520 llvm::Type *Ty = Arg->getType();
9521 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::aarch64_frint32x, Ty),
9525 if (BuiltinID == clang::AArch64::BI__builtin_arm_frint64xf ||
9526 BuiltinID == clang::AArch64::BI__builtin_arm_frint64x) {
9528 llvm::Type *Ty = Arg->getType();
9529 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::aarch64_frint64x, Ty),
9533 if (BuiltinID == clang::AArch64::BI__builtin_arm_jcvt) {
9535 "__jcvt of unusual size!");
9537 return Builder.CreateCall(
9541 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b ||
9542 BuiltinID == clang::AArch64::BI__builtin_arm_st64b ||
9543 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv ||
9544 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv0) {
9548 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b) {
9552 llvm::Value *Val = Builder.CreateCall(F, MemAddr);
9554 for (
size_t i = 0; i < 8; i++) {
9555 llvm::Value *ValOffsetPtr =
9556 Builder.CreateGEP(
Int64Ty, ValPtr, Builder.getInt32(i));
9559 ToRet = Builder.CreateStore(Builder.CreateExtractValue(Val, i), Addr);
9566 Args.push_back(MemAddr);
9567 for (
size_t i = 0; i < 8; i++) {
9568 llvm::Value *ValOffsetPtr =
9569 Builder.CreateGEP(
Int64Ty, ValPtr, Builder.getInt32(i));
9572 Args.push_back(Builder.CreateLoad(Addr));
9575 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_st64b
9576 ? Intrinsic::aarch64_st64b
9577 : BuiltinID == clang::AArch64::BI__builtin_arm_st64bv
9578 ? Intrinsic::aarch64_st64bv
9579 : Intrinsic::aarch64_st64bv0);
9581 return Builder.CreateCall(F, Args);
9585 if (BuiltinID == clang::AArch64::BI__builtin_arm_rndr ||
9586 BuiltinID == clang::AArch64::BI__builtin_arm_rndrrs) {
9588 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_rndr
9589 ? Intrinsic::aarch64_rndr
9590 : Intrinsic::aarch64_rndrrs);
9592 llvm::Value *Val = Builder.CreateCall(F);
9593 Value *RandomValue = Builder.CreateExtractValue(Val, 0);
9594 Value *Status = Builder.CreateExtractValue(Val, 1);
9597 Builder.CreateStore(RandomValue, MemAddress);
9598 Status = Builder.CreateZExt(Status,
Int32Ty);
9602 if (BuiltinID == clang::AArch64::BI__clear_cache) {
9603 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
9606 for (
unsigned i = 0; i < 2; i++)
9610 StringRef Name = FD->
getName();
9614 if ((BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
9615 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) &&
9619 ? Intrinsic::aarch64_ldaxp
9620 : Intrinsic::aarch64_ldxp);
9623 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr,
Int8PtrTy),
9626 Value *Val0 = Builder.CreateExtractValue(Val, 1);
9627 Value *Val1 = Builder.CreateExtractValue(Val, 0);
9628 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
9629 Val0 = Builder.CreateZExt(Val0, Int128Ty);
9630 Val1 = Builder.CreateZExt(Val1, Int128Ty);
9632 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
9633 Val = Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
9634 Val = Builder.CreateOr(Val, Val1);
9636 }
else if (BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
9637 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) {
9644 llvm::Type *PtrTy =
IntTy->getPointerTo();
9645 LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy);
9649 ? Intrinsic::aarch64_ldaxr
9650 : Intrinsic::aarch64_ldxr,
9652 CallInst *Val = Builder.CreateCall(F, LoadAddr,
"ldxr");
9656 if (RealResTy->isPointerTy())
9657 return Builder.CreateIntToPtr(Val, RealResTy);
9659 llvm::Type *IntResTy = llvm::IntegerType::get(
9661 return Builder.CreateBitCast(Builder.CreateTruncOrBitCast(Val, IntResTy),
9665 if ((BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
9666 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) &&
9670 ? Intrinsic::aarch64_stlxp
9671 : Intrinsic::aarch64_stxp);
9677 Tmp = Builder.CreateElementBitCast(Tmp, STy);
9678 llvm::Value *Val = Builder.CreateLoad(Tmp);
9680 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
9681 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
9684 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"stxp");
9687 if (BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
9688 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) {
9695 StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo());
9697 if (StoreVal->getType()->isPointerTy())
9698 StoreVal = Builder.CreatePtrToInt(StoreVal,
Int64Ty);
9700 llvm::Type *
IntTy = llvm::IntegerType::get(
9703 StoreVal = Builder.CreateBitCast(StoreVal,
IntTy);
9704 StoreVal = Builder.CreateZExtOrBitCast(StoreVal,
Int64Ty);
9709 ? Intrinsic::aarch64_stlxr
9710 : Intrinsic::aarch64_stxr,
9711 StoreAddr->getType());
9712 CallInst *CI = Builder.CreateCall(F, {StoreVal, StoreAddr},
"stxr");
9714 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
9718 if (BuiltinID == clang::AArch64::BI__getReg) {
9721 llvm_unreachable(
"Sema will ensure that the parameter is constant");
9727 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)};
9728 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
9729 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
9733 return Builder.CreateCall(F, Metadata);
9736 if (BuiltinID == clang::AArch64::BI__break) {
9739 llvm_unreachable(
"Sema will ensure that the parameter is constant");
9745 if (BuiltinID == clang::AArch64::BI__builtin_arm_clrex) {
9747 return Builder.CreateCall(F);
9750 if (BuiltinID == clang::AArch64::BI_ReadWriteBarrier)
9751 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
9752 llvm::SyncScope::SingleThread);
9755 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
9756 switch (BuiltinID) {
9757 case clang::AArch64::BI__builtin_arm_crc32b:
9758 CRCIntrinsicID = Intrinsic::aarch64_crc32b;
break;
9759 case clang::AArch64::BI__builtin_arm_crc32cb:
9760 CRCIntrinsicID = Intrinsic::aarch64_crc32cb;
break;
9761 case clang::AArch64::BI__builtin_arm_crc32h:
9762 CRCIntrinsicID = Intrinsic::aarch64_crc32h;
break;
9763 case clang::AArch64::BI__builtin_arm_crc32ch:
9764 CRCIntrinsicID = Intrinsic::aarch64_crc32ch;
break;
9765 case clang::AArch64::BI__builtin_arm_crc32w:
9766 CRCIntrinsicID = Intrinsic::aarch64_crc32w;
break;
9767 case clang::AArch64::BI__builtin_arm_crc32cw:
9768 CRCIntrinsicID = Intrinsic::aarch64_crc32cw;
break;
9769 case clang::AArch64::BI__builtin_arm_crc32d:
9770 CRCIntrinsicID = Intrinsic::aarch64_crc32x;
break;
9771 case clang::AArch64::BI__builtin_arm_crc32cd:
9772 CRCIntrinsicID = Intrinsic::aarch64_crc32cx;
break;
9775 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
9780 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
9781 Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy);
9783 return Builder.CreateCall(F, {Arg0, Arg1});
9787 if (BuiltinID == AArch64::BI__builtin_arm_mops_memset_tag) {
9791 Dst = Builder.CreatePointerCast(Dst,
Int8PtrTy);
9792 Val = Builder.CreateTrunc(Val,
Int8Ty);
9793 Size = Builder.CreateIntCast(Size,
Int64Ty,
false);
9794 return Builder.CreateCall(
9795 CGM.
getIntrinsic(Intrinsic::aarch64_mops_memset_tag), {Dst, Val, Size});
9799 Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
9800 switch (BuiltinID) {
9801 case clang::AArch64::BI__builtin_arm_irg:
9802 MTEIntrinsicID = Intrinsic::aarch64_irg;
break;
9803 case clang::AArch64::BI__builtin_arm_addg:
9804 MTEIntrinsicID = Intrinsic::aarch64_addg;
break;
9805 case clang::AArch64::BI__builtin_arm_gmi:
9806 MTEIntrinsicID = Intrinsic::aarch64_gmi;
break;
9807 case clang::AArch64::BI__builtin_arm_ldg:
9808 MTEIntrinsicID = Intrinsic::aarch64_ldg;
break;
9809 case clang::AArch64::BI__builtin_arm_stg:
9810 MTEIntrinsicID = Intrinsic::aarch64_stg;
break;
9811 case clang::AArch64::BI__builtin_arm_subp:
9812 MTEIntrinsicID = Intrinsic::aarch64_subp;
break;
9815 if (MTEIntrinsicID != Intrinsic::not_intrinsic) {
9818 if (MTEIntrinsicID == Intrinsic::aarch64_irg) {
9823 Mask = Builder.CreateZExt(Mask,
Int64Ty);
9824 Value *RV = Builder.CreateCall(
9826 return Builder.CreatePointerCast(RV, T);
9828 if (MTEIntrinsicID == Intrinsic::aarch64_addg) {
9833 TagOffset = Builder.CreateZExt(TagOffset,
Int64Ty);
9834 Value *RV = Builder.CreateCall(
9836 return Builder.CreatePointerCast(RV, T);
9838 if (MTEIntrinsicID == Intrinsic::aarch64_gmi) {
9842 ExcludedMask = Builder.CreateZExt(ExcludedMask,
Int64Ty);
9844 return Builder.CreateCall(
9850 if (MTEIntrinsicID == Intrinsic::aarch64_ldg) {
9852 TagAddress = Builder.CreatePointerCast(TagAddress,
Int8PtrTy);
9853 Value *RV = Builder.CreateCall(
9855 return Builder.CreatePointerCast(RV, T);
9860 if (MTEIntrinsicID == Intrinsic::aarch64_stg) {
9862 TagAddress = Builder.CreatePointerCast(TagAddress,
Int8PtrTy);
9863 return Builder.CreateCall(
9866 if (MTEIntrinsicID == Intrinsic::aarch64_subp) {
9869 PointerA = Builder.CreatePointerCast(PointerA,
Int8PtrTy);
9870 PointerB = Builder.CreatePointerCast(PointerB,
Int8PtrTy);
9871 return Builder.CreateCall(
9876 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
9877 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
9878 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
9879 BuiltinID == clang::AArch64::BI__builtin_arm_wsr ||
9880 BuiltinID == clang::AArch64::BI__builtin_arm_wsr64 ||
9881 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp) {
9884 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
9885 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
9886 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp)
9889 bool IsPointerBuiltin = BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
9890 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp;
9892 bool Is64Bit = BuiltinID != clang::AArch64::BI__builtin_arm_rsr &&
9893 BuiltinID != clang::AArch64::BI__builtin_arm_wsr;
9895 llvm::Type *ValueType;
9896 llvm::Type *RegisterType =
Int64Ty;
9897 if (IsPointerBuiltin) {
9899 }
else if (Is64Bit) {
9909 if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
9910 BuiltinID == clang::AArch64::BI_WriteStatusReg) {
9916 std::string SysRegStr;
9917 llvm::raw_string_ostream(SysRegStr) <<
9918 ((1 << 1) | ((SysReg >> 14) & 1)) <<
":" <<
9919 ((SysReg >> 11) & 7) <<
":" <<
9920 ((SysReg >> 7) & 15) <<
":" <<
9921 ((SysReg >> 3) & 15) <<
":" <<
9924 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) };
9925 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
9926 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
9928 llvm::Type *RegisterType =
Int64Ty;
9929 llvm::Type *Types[] = { RegisterType };
9931 if (BuiltinID == clang::AArch64::BI_ReadStatusReg) {
9932 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::read_register, Types);
9934 return Builder.CreateCall(F, Metadata);
9937 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
9940 return Builder.CreateCall(F, { Metadata, ArgValue });
9943 if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
9946 return Builder.CreateCall(F);
9949 if (BuiltinID == clang::AArch64::BI__builtin_sponentry) {
9951 return Builder.CreateCall(F);
9954 if (BuiltinID == clang::AArch64::BI__mulh ||
9955 BuiltinID == clang::AArch64::BI__umulh) {
9957 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
9959 bool IsSigned = BuiltinID == clang::AArch64::BI__mulh;
9965 Value *MulResult, *HigherBits;
9967 MulResult = Builder.CreateNSWMul(LHS, RHS);
9968 HigherBits = Builder.CreateAShr(MulResult, 64);
9970 MulResult = Builder.CreateNUWMul(LHS, RHS);
9971 HigherBits = Builder.CreateLShr(MulResult, 64);
9973 HigherBits = Builder.CreateIntCast(HigherBits, ResType, IsSigned);
9978 if (BuiltinID == AArch64::BI__writex18byte ||
9979 BuiltinID == AArch64::BI__writex18word ||
9980 BuiltinID == AArch64::BI__writex18dword ||
9981 BuiltinID == AArch64::BI__writex18qword) {
9986 llvm::Metadata *Ops[] = {llvm::MDString::get(Context,
"x18")};
9987 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
9988 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
9991 llvm::Value *X18 = Builder.CreateCall(F, Metadata);
9992 X18 = Builder.CreateIntToPtr(X18, llvm::PointerType::get(
Int8Ty, 0));
9997 Ptr = Builder.CreatePointerCast(Ptr, llvm::PointerType::get(
IntTy, 0));
10003 if (BuiltinID == AArch64::BI__readx18byte ||
10004 BuiltinID == AArch64::BI__readx18word ||
10005 BuiltinID == AArch64::BI__readx18dword ||
10006 BuiltinID == AArch64::BI__readx18qword) {
10011 llvm::Metadata *Ops[] = {llvm::MDString::get(Context,
"x18")};
10012 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
10013 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
10014 llvm::Function *F =
10016 llvm::Value *X18 = Builder.CreateCall(F, Metadata);
10017 X18 = Builder.CreateIntToPtr(X18, llvm::PointerType::get(
Int8Ty, 0));
10022 Ptr = Builder.CreatePointerCast(Ptr, llvm::PointerType::get(
IntTy, 0));
10034 unsigned ICEArguments = 0;
10041 for (
unsigned i = 0, e = E->
getNumArgs() - 1; i != e; i++) {
10043 switch (BuiltinID) {
10044 case NEON::BI__builtin_neon_vld1_v:
10045 case NEON::BI__builtin_neon_vld1q_v:
10046 case NEON::BI__builtin_neon_vld1_dup_v:
10047 case NEON::BI__builtin_neon_vld1q_dup_v:
10048 case NEON::BI__builtin_neon_vld1_lane_v:
10049 case NEON::BI__builtin_neon_vld1q_lane_v:
10050 case NEON::BI__builtin_neon_vst1_v:
10051 case NEON::BI__builtin_neon_vst1q_v:
10052 case NEON::BI__builtin_neon_vst1_lane_v:
10053 case NEON::BI__builtin_neon_vst1q_lane_v:
10061 if ((ICEArguments & (1 << i)) == 0) {
10066 Ops.push_back(llvm::ConstantInt::get(
10079 assert(
Result &&
"SISD intrinsic should have been handled");
10089 bool usgn =
Type.isUnsigned();
10090 bool quad =
Type.isQuad();
10093 switch (BuiltinID) {
10095 case NEON::BI__builtin_neon_vabsh_f16:
10098 case NEON::BI__builtin_neon_vaddq_p128: {
10101 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
10102 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
10103 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]);
10104 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
10105 return Builder.CreateBitCast(Ops[0], Int128Ty);
10107 case NEON::BI__builtin_neon_vldrq_p128: {
10108 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
10109 llvm::Type *Int128PTy = llvm::PointerType::get(Int128Ty, 0);
10111 return Builder.CreateAlignedLoad(Int128Ty, Ptr,
10114 case NEON::BI__builtin_neon_vstrq_p128: {
10115 llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(
getLLVMContext(), 128);
10116 Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy);
10119 case NEON::BI__builtin_neon_vcvts_f32_u32:
10120 case NEON::BI__builtin_neon_vcvtd_f64_u64:
10123 case NEON::BI__builtin_neon_vcvts_f32_s32:
10124 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
10126 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
10129 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
10131 return Builder.CreateUIToFP(Ops[0], FTy);
10132 return Builder.CreateSIToFP(Ops[0], FTy);
10134 case NEON::BI__builtin_neon_vcvth_f16_u16:
10135 case NEON::BI__builtin_neon_vcvth_f16_u32:
10136 case NEON::BI__builtin_neon_vcvth_f16_u64:
10139 case NEON::BI__builtin_neon_vcvth_f16_s16:
10140 case NEON::BI__builtin_neon_vcvth_f16_s32:
10141 case NEON::BI__builtin_neon_vcvth_f16_s64: {
10143 llvm::Type *FTy =
HalfTy;
10145 if (Ops[0]->
getType()->getPrimitiveSizeInBits() == 64)
10147 else if (Ops[0]->
getType()->getPrimitiveSizeInBits() == 32)
10151 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
10153 return Builder.CreateUIToFP(Ops[0], FTy);
10154 return Builder.CreateSIToFP(Ops[0], FTy);
10156 case NEON::BI__builtin_neon_vcvtah_u16_f16:
10157 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
10158 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
10159 case NEON::BI__builtin_neon_vcvtph_u16_f16:
10160 case NEON::BI__builtin_neon_vcvth_u16_f16:
10161 case NEON::BI__builtin_neon_vcvtah_s16_f16:
10162 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
10163 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
10164 case NEON::BI__builtin_neon_vcvtph_s16_f16:
10165 case NEON::BI__builtin_neon_vcvth_s16_f16: {
10168 llvm::Type* FTy =
HalfTy;
10169 llvm::Type *Tys[2] = {InTy, FTy};
10171 switch (BuiltinID) {
10172 default: llvm_unreachable(
"missing builtin ID in switch!");
10173 case NEON::BI__builtin_neon_vcvtah_u16_f16:
10174 Int = Intrinsic::aarch64_neon_fcvtau;
break;
10175 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
10176 Int = Intrinsic::aarch64_neon_fcvtmu;
break;
10177 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
10178 Int = Intrinsic::aarch64_neon_fcvtnu;
break;
10179 case NEON::BI__builtin_neon_vcvtph_u16_f16:
10180 Int = Intrinsic::aarch64_neon_fcvtpu;
break;
10181 case NEON::BI__builtin_neon_vcvth_u16_f16:
10182 Int = Intrinsic::aarch64_neon_fcvtzu;
break;
10183 case NEON::BI__builtin_neon_vcvtah_s16_f16:
10184 Int = Intrinsic::aarch64_neon_fcvtas;
break;
10185 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
10186 Int = Intrinsic::aarch64_neon_fcvtms;
break;
10187 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
10188 Int = Intrinsic::aarch64_neon_fcvtns;
break;
10189 case NEON::BI__builtin_neon_vcvtph_s16_f16:
10190 Int = Intrinsic::aarch64_neon_fcvtps;
break;
10191 case NEON::BI__builtin_neon_vcvth_s16_f16:
10192 Int = Intrinsic::aarch64_neon_fcvtzs;
break;
10195 return Builder.CreateTrunc(Ops[0],
Int16Ty);
10197 case NEON::BI__builtin_neon_vcaleh_f16:
10198 case NEON::BI__builtin_neon_vcalth_f16:
10199 case NEON::BI__builtin_neon_vcageh_f16:
10200 case NEON::BI__builtin_neon_vcagth_f16: {
10203 llvm::Type* FTy =
HalfTy;
10204 llvm::Type *Tys[2] = {InTy, FTy};
10206 switch (BuiltinID) {
10207 default: llvm_unreachable(
"missing builtin ID in switch!");
10208 case NEON::BI__builtin_neon_vcageh_f16:
10209 Int = Intrinsic::aarch64_neon_facge;
break;
10210 case NEON::BI__builtin_neon_vcagth_f16:
10211 Int = Intrinsic::aarch64_neon_facgt;
break;
10212 case NEON::BI__builtin_neon_vcaleh_f16:
10213 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]);
break;
10214 case NEON::BI__builtin_neon_vcalth_f16:
10215 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]);
break;
10218 return Builder.CreateTrunc(Ops[0],
Int16Ty);
10220 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
10221 case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
10224 llvm::Type* FTy =
HalfTy;
10225 llvm::Type *Tys[2] = {InTy, FTy};
10227 switch (BuiltinID) {
10228 default: llvm_unreachable(
"missing builtin ID in switch!");
10229 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
10230 Int = Intrinsic::aarch64_neon_vcvtfp2fxs;
break;
10231 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
10232 Int = Intrinsic::aarch64_neon_vcvtfp2fxu;
break;
10235 return Builder.CreateTrunc(Ops[0],
Int16Ty);
10237 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
10238 case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
10240 llvm::Type* FTy =
HalfTy;
10242 llvm::Type *Tys[2] = {FTy, InTy};
10244 switch (BuiltinID) {
10245 default: llvm_unreachable(
"missing builtin ID in switch!");
10246 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
10247 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
10248 Ops[0] = Builder.CreateSExt(Ops[0], InTy,
"sext");
10250 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
10251 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
10252 Ops[0] = Builder.CreateZExt(Ops[0], InTy);
10257 case NEON::BI__builtin_neon_vpaddd_s64: {
10258 auto *Ty = llvm::FixedVectorType::get(
Int64Ty, 2);
10261 Vec = Builder.CreateBitCast(Vec, Ty,
"v2i64");
10262 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
10263 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
10264 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0,
"lane0");
10265 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1,
"lane1");
10267 return Builder.CreateAdd(Op0, Op1,
"vpaddd");
10269 case NEON::BI__builtin_neon_vpaddd_f64: {
10270 auto *Ty = llvm::FixedVectorType::get(
DoubleTy, 2);
10273 Vec = Builder.CreateBitCast(Vec, Ty,
"v2f64");
10274 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
10275 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
10276 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0,
"lane0");
10277 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1,
"lane1");
10279 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
10281 case NEON::BI__builtin_neon_vpadds_f32: {
10282 auto *Ty = llvm::FixedVectorType::get(
FloatTy, 2);
10285 Vec = Builder.CreateBitCast(Vec, Ty,
"v2f32");
10286 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
10287 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
10288 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0,
"lane0");
10289 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1,
"lane1");
10291 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
10293 case NEON::BI__builtin_neon_vceqzd_s64:
10294 case NEON::BI__builtin_neon_vceqzd_f64:
10295 case NEON::BI__builtin_neon_vceqzs_f32:
10296 case NEON::BI__builtin_neon_vceqzh_f16:
10300 ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ,
"vceqz");
10301 case NEON::BI__builtin_neon_vcgezd_s64:
10302 case NEON::BI__builtin_neon_vcgezd_f64:
10303 case NEON::BI__builtin_neon_vcgezs_f32:
10304 case NEON::BI__builtin_neon_vcgezh_f16:
10308 ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE,
"vcgez");
10309 case NEON::BI__builtin_neon_vclezd_s64:
10310 case NEON::BI__builtin_neon_vclezd_f64:
10311 case NEON::BI__builtin_neon_vclezs_f32:
10312 case NEON::BI__builtin_neon_vclezh_f16:
10316 ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE,
"vclez");
10317 case NEON::BI__builtin_neon_vcgtzd_s64:
10318 case NEON::BI__builtin_neon_vcgtzd_f64:
10319 case NEON::BI__builtin_neon_vcgtzs_f32:
10320 case NEON::BI__builtin_neon_vcgtzh_f16:
10324 ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT,
"vcgtz");
10325 case NEON::BI__builtin_neon_vcltzd_s64:
10326 case NEON::BI__builtin_neon_vcltzd_f64:
10327 case NEON::BI__builtin_neon_vcltzs_f32:
10328 case NEON::BI__builtin_neon_vcltzh_f16:
10332 ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT,
"vcltz");
10334 case NEON::BI__builtin_neon_vceqzd_u64: {
10336 Ops[0] = Builder.CreateBitCast(Ops[0],
Int64Ty);
10338 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(
Int64Ty));
10339 return Builder.CreateSExt(Ops[0],
Int64Ty,
"vceqzd");
10341 case NEON::BI__builtin_neon_vceqd_f64:
10342 case NEON::BI__builtin_neon_vcled_f64:
10343 case NEON::BI__builtin_neon_vcltd_f64:
10344 case NEON::BI__builtin_neon_vcged_f64:
10345 case NEON::BI__builtin_neon_vcgtd_f64: {
10346 llvm::CmpInst::Predicate
P;
10347 switch (BuiltinID) {
10348 default: llvm_unreachable(
"missing builtin ID in switch!");
10349 case NEON::BI__builtin_neon_vceqd_f64:
P = llvm::FCmpInst::FCMP_OEQ;
break;
10350 case NEON::BI__builtin_neon_vcled_f64:
P = llvm::FCmpInst::FCMP_OLE;
break;
10351 case NEON::BI__builtin_neon_vcltd_f64:
P = llvm::FCmpInst::FCMP_OLT;
break;
10352 case NEON::BI__builtin_neon_vcged_f64:
P = llvm::FCmpInst::FCMP_OGE;
break;
10353 case NEON::BI__builtin_neon_vcgtd_f64:
P = llvm::FCmpInst::FCMP_OGT;
break;
10356 Ops[0] = Builder.CreateBitCast(Ops[0],
DoubleTy);
10357 Ops[1] = Builder.CreateBitCast(Ops[1],
DoubleTy);
10358 if (
P == llvm::FCmpInst::FCMP_OEQ)
10359 Ops[0] = Builder.CreateFCmp(
P, Ops[0], Ops[1]);
10361 Ops[0] = Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
10362 return Builder.CreateSExt(Ops[0],
Int64Ty,
"vcmpd");
10364 case NEON::BI__builtin_neon_vceqs_f32:
10365 case NEON::BI__builtin_neon_vcles_f32:
10366 case NEON::BI__builtin_neon_vclts_f32:
10367 case NEON::BI__builtin_neon_vcges_f32:
10368 case NEON::BI__builtin_neon_vcgts_f32: {
10369 llvm::CmpInst::Predicate
P;
10370 switch (BuiltinID) {
10371 default: llvm_unreachable(
"missing builtin ID in switch!");
10372 case NEON::BI__builtin_neon_vceqs_f32:
P = llvm::FCmpInst::FCMP_OEQ;
break;
10373 case NEON::BI__builtin_neon_vcles_f32:
P = llvm::FCmpInst::FCMP_OLE;
break;
10374 case NEON::BI__builtin_neon_vclts_f32:
P = llvm::FCmpInst::FCMP_OLT;
break;
10375 case NEON::BI__builtin_neon_vcges_f32:
P = llvm::FCmpInst::FCMP_OGE;
break;
10376 case NEON::BI__builtin_neon_vcgts_f32:
P = llvm::FCmpInst::FCMP_OGT;
break;
10379 Ops[0] = Builder.CreateBitCast(Ops[0],
FloatTy);
10380 Ops[1] = Builder.CreateBitCast(Ops[1],
FloatTy);
10381 if (
P == llvm::FCmpInst::FCMP_OEQ)
10382 Ops[0] = Builder.CreateFCmp(
P, Ops[0], Ops[1]);
10384 Ops[0] = Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
10385 return Builder.CreateSExt(Ops[0],
Int32Ty,
"vcmpd");
10387 case NEON::BI__builtin_neon_vceqh_f16:
10388 case NEON::BI__builtin_neon_vcleh_f16:
10389 case NEON::BI__builtin_neon_vclth_f16:
10390 case NEON::BI__builtin_neon_vcgeh_f16:
10391 case NEON::BI__builtin_neon_vcgth_f16: {
10392 llvm::CmpInst::Predicate
P;
10393 switch (BuiltinID) {
10394 default: llvm_unreachable(
"missing builtin ID in switch!");
10395 case NEON::BI__builtin_neon_vceqh_f16:
P = llvm::FCmpInst::FCMP_OEQ;
break;
10396 case NEON::BI__builtin_neon_vcleh_f16:
P = llvm::FCmpInst::FCMP_OLE;
break;
10397 case NEON::BI__builtin_neon_vclth_f16:
P = llvm::FCmpInst::FCMP_OLT;
break;
10398 case NEON::BI__builtin_neon_vcgeh_f16:
P = llvm::FCmpInst::FCMP_OGE;
break;
10399 case NEON::BI__builtin_neon_vcgth_f16:
P = llvm::FCmpInst::FCMP_OGT;
break;
10402 Ops[0] = Builder.CreateBitCast(Ops[0],
HalfTy);
10403 Ops[1] = Builder.CreateBitCast(Ops[1],
HalfTy);
10404 if (
P == llvm::FCmpInst::FCMP_OEQ)
10405 Ops[0] = Builder.CreateFCmp(
P, Ops[0], Ops[1]);
10407 Ops[0] = Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
10408 return Builder.CreateSExt(Ops[0],
Int16Ty,
"vcmpd");
10410 case NEON::BI__builtin_neon_vceqd_s64:
10411 case NEON::BI__builtin_neon_vceqd_u64:
10412 case NEON::BI__builtin_neon_vcgtd_s64:
10413 case NEON::BI__builtin_neon_vcgtd_u64:
10414 case NEON::BI__builtin_neon_vcltd_s64:
10415 case NEON::BI__builtin_neon_vcltd_u64:
10416 case NEON::BI__builtin_neon_vcged_u64:
10417 case NEON::BI__builtin_neon_vcged_s64:
10418 case NEON::BI__builtin_neon_vcled_u64:
10419 case NEON::BI__builtin_neon_vcled_s64: {
10420 llvm::CmpInst::Predicate
P;
10421 switch (BuiltinID) {
10422 default: llvm_unreachable(
"missing builtin ID in switch!");
10423 case NEON::BI__builtin_neon_vceqd_s64:
10424 case NEON::BI__builtin_neon_vceqd_u64:
P = llvm::ICmpInst::ICMP_EQ;
break;
10425 case NEON::BI__builtin_neon_vcgtd_s64:
P = llvm::ICmpInst::ICMP_SGT;
break;
10426 case NEON::BI__builtin_neon_vcgtd_u64:
P = llvm::ICmpInst::ICMP_UGT;
break;
10427 case NEON::BI__builtin_neon_vcltd_s64:
P = llvm::ICmpInst::ICMP_SLT;
break;
10428 case NEON::BI__builtin_neon_vcltd_u64:
P = llvm::ICmpInst::ICMP_ULT;
break;
10429 case NEON::BI__builtin_neon_vcged_u64:
P = llvm::ICmpInst::ICMP_UGE;
break;
10430 case NEON::BI__builtin_neon_vcged_s64:
P = llvm::ICmpInst::ICMP_SGE;
break;
10431 case NEON::BI__builtin_neon_vcled_u64:
P = llvm::ICmpInst::ICMP_ULE;
break;
10432 case NEON::BI__builtin_neon_vcled_s64:
P = llvm::ICmpInst::ICMP_SLE;
break;
10435 Ops[0] = Builder.CreateBitCast(Ops[0],
Int64Ty);
10436 Ops[1] = Builder.CreateBitCast(Ops[1],
Int64Ty);
10437 Ops[0] = Builder.CreateICmp(
P, Ops[0], Ops[1]);
10438 return Builder.CreateSExt(Ops[0],
Int64Ty,
"vceqd");
10440 case NEON::BI__builtin_neon_vtstd_s64:
10441 case NEON::BI__builtin_neon_vtstd_u64: {
10443 Ops[0] = Builder.CreateBitCast(Ops[0],
Int64Ty);
10444 Ops[1] = Builder.CreateBitCast(Ops[1],
Int64Ty);
10445 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
10446 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
10447 llvm::Constant::getNullValue(
Int64Ty));
10448 return Builder.CreateSExt(Ops[0],
Int64Ty,
"vtstd");
10450 case NEON::BI__builtin_neon_vset_lane_i8:
10451 case NEON::BI__builtin_neon_vset_lane_i16:
10452 case NEON::BI__builtin_neon_vset_lane_i32:
10453 case NEON::BI__builtin_neon_vset_lane_i64:
10454 case NEON::BI__builtin_neon_vset_lane_bf16:
10455 case NEON::BI__builtin_neon_vset_lane_f32:
10456 case NEON::BI__builtin_neon_vsetq_lane_i8:
10457 case NEON::BI__builtin_neon_vsetq_lane_i16:
10458 case NEON::BI__builtin_neon_vsetq_lane_i32:
10459 case NEON::BI__builtin_neon_vsetq_lane_i64:
10460 case NEON::BI__builtin_neon_vsetq_lane_bf16:
10461 case NEON::BI__builtin_neon_vsetq_lane_f32:
10463 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
10464 case NEON::BI__builtin_neon_vset_lane_f64:
10467 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 1));
10469 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
10470 case NEON::BI__builtin_neon_vsetq_lane_f64:
10473 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 2));
10475 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
10477 case NEON::BI__builtin_neon_vget_lane_i8:
10478 case NEON::BI__builtin_neon_vdupb_lane_i8:
10480 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int8Ty, 8));
10483 case NEON::BI__builtin_neon_vgetq_lane_i8:
10484 case NEON::BI__builtin_neon_vdupb_laneq_i8:
10486 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int8Ty, 16));
10489 case NEON::BI__builtin_neon_vget_lane_i16:
10490 case NEON::BI__builtin_neon_vduph_lane_i16:
10492 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int16Ty, 4));
10495 case NEON::BI__builtin_neon_vgetq_lane_i16:
10496 case NEON::BI__builtin_neon_vduph_laneq_i16:
10498 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int16Ty, 8));
10501 case NEON::BI__builtin_neon_vget_lane_i32:
10502 case NEON::BI__builtin_neon_vdups_lane_i32:
10504 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int32Ty, 2));
10507 case NEON::BI__builtin_neon_vdups_lane_f32:
10509 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 2));
10512 case NEON::BI__builtin_neon_vgetq_lane_i32:
10513 case NEON::BI__builtin_neon_vdups_laneq_i32:
10515 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int32Ty, 4));
10518 case NEON::BI__builtin_neon_vget_lane_i64:
10519 case NEON::BI__builtin_neon_vdupd_lane_i64:
10521 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int64Ty, 1));
10524 case NEON::BI__builtin_neon_vdupd_lane_f64:
10526 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 1));
10529 case NEON::BI__builtin_neon_vgetq_lane_i64:
10530 case NEON::BI__builtin_neon_vdupd_laneq_i64:
10532 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int64Ty, 2));
10535 case NEON::BI__builtin_neon_vget_lane_f32:
10537 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 2));
10540 case NEON::BI__builtin_neon_vget_lane_f64:
10542 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 1));
10545 case NEON::BI__builtin_neon_vgetq_lane_f32:
10546 case NEON::BI__builtin_neon_vdups_laneq_f32:
10548 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 4));
10551 case NEON::BI__builtin_neon_vgetq_lane_f64:
10552 case NEON::BI__builtin_neon_vdupd_laneq_f64:
10554 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 2));
10557 case NEON::BI__builtin_neon_vaddh_f16:
10559 return Builder.CreateFAdd(Ops[0], Ops[1],
"vaddh");
10560 case NEON::BI__builtin_neon_vsubh_f16:
10562 return Builder.CreateFSub(Ops[0], Ops[1],
"vsubh");
10563 case NEON::BI__builtin_neon_vmulh_f16:
10565 return Builder.CreateFMul(Ops[0], Ops[1],
"vmulh");
10566 case NEON::BI__builtin_neon_vdivh_f16:
10568 return Builder.CreateFDiv(Ops[0], Ops[1],
"vdivh");
10569 case NEON::BI__builtin_neon_vfmah_f16:
10572 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
10574 case NEON::BI__builtin_neon_vfmsh_f16: {
10581 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
10584 case NEON::BI__builtin_neon_vaddd_s64:
10585 case NEON::BI__builtin_neon_vaddd_u64:
10587 case NEON::BI__builtin_neon_vsubd_s64:
10588 case NEON::BI__builtin_neon_vsubd_u64:
10590 case NEON::BI__builtin_neon_vqdmlalh_s16:
10591 case NEON::BI__builtin_neon_vqdmlslh_s16: {
10595 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
10597 ProductOps,
"vqdmlXl");
10598 Constant *CI = ConstantInt::get(
SizeTy, 0);
10599 Ops[1] = Builder.CreateExtractElement(Ops[1], CI,
"lane0");
10601 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
10602 ? Intrinsic::aarch64_neon_sqadd
10603 : Intrinsic::aarch64_neon_sqsub;
10606 case NEON::BI__builtin_neon_vqshlud_n_s64: {
10608 Ops[1] = Builder.CreateZExt(Ops[1],
Int64Ty);
10612 case NEON::BI__builtin_neon_vqshld_n_u64:
10613 case NEON::BI__builtin_neon_vqshld_n_s64: {
10614 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
10615 ? Intrinsic::aarch64_neon_uqshl
10616 : Intrinsic::aarch64_neon_sqshl;
10618 Ops[1] = Builder.CreateZExt(Ops[1],
Int64Ty);
10621 case NEON::BI__builtin_neon_vrshrd_n_u64:
10622 case NEON::BI__builtin_neon_vrshrd_n_s64: {
10623 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
10624 ? Intrinsic::aarch64_neon_urshl
10625 : Intrinsic::aarch64_neon_srshl;
10628 Ops[1] = ConstantInt::get(
Int64Ty, -SV);
10631 case NEON::BI__builtin_neon_vrsrad_n_u64:
10632 case NEON::BI__builtin_neon_vrsrad_n_s64: {
10633 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
10634 ? Intrinsic::aarch64_neon_urshl
10635 : Intrinsic::aarch64_neon_srshl;
10636 Ops[1] = Builder.CreateBitCast(Ops[1],
Int64Ty);
10639 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
10640 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1],
Int64Ty));
10642 case NEON::BI__builtin_neon_vshld_n_s64:
10643 case NEON::BI__builtin_neon_vshld_n_u64: {
10645 return Builder.CreateShl(
10646 Ops[0], ConstantInt::get(
Int64Ty, Amt->getZExtValue()),
"shld_n");
10648 case NEON::BI__builtin_neon_vshrd_n_s64: {
10650 return Builder.CreateAShr(
10651 Ops[0], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t>(63),
10652 Amt->getZExtValue())),
10655 case NEON::BI__builtin_neon_vshrd_n_u64: {
10657 uint64_t ShiftAmt = Amt->getZExtValue();
10659 if (ShiftAmt == 64)
10660 return ConstantInt::get(
Int64Ty, 0);
10661 return Builder.CreateLShr(Ops[0], ConstantInt::get(
Int64Ty, ShiftAmt),
10664 case NEON::BI__builtin_neon_vsrad_n_s64: {
10666 Ops[1] = Builder.CreateAShr(
10667 Ops[1], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t>(63),
10668 Amt->getZExtValue())),
10670 return Builder.CreateAdd(Ops[0], Ops[1]);
10672 case NEON::BI__builtin_neon_vsrad_n_u64: {
10674 uint64_t ShiftAmt = Amt->getZExtValue();
10677 if (ShiftAmt == 64)
10679 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(
Int64Ty, ShiftAmt),
10681 return Builder.CreateAdd(Ops[0], Ops[1]);
10683 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
10684 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
10685 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
10686 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
10692 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
10694 ProductOps,
"vqdmlXl");
10695 Constant *CI = ConstantInt::get(
SizeTy, 0);
10696 Ops[1] = Builder.CreateExtractElement(Ops[1], CI,
"lane0");
10699 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
10700 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
10701 ? Intrinsic::aarch64_neon_sqadd
10702 : Intrinsic::aarch64_neon_sqsub;
10705 case NEON::BI__builtin_neon_vqdmlals_s32:
10706 case NEON::BI__builtin_neon_vqdmlsls_s32: {
10708 ProductOps.push_back(Ops[1]);
10712 ProductOps,
"vqdmlXl");
10714 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
10715 ? Intrinsic::aarch64_neon_sqadd
10716 : Intrinsic::aarch64_neon_sqsub;
10719 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
10720 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
10721 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
10722 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
10726 ProductOps.push_back(Ops[1]);
10727 ProductOps.push_back(Ops[2]);
10730 ProductOps,
"vqdmlXl");
10733 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
10734 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
10735 ? Intrinsic::aarch64_neon_sqadd
10736 : Intrinsic::aarch64_neon_sqsub;
10739 case NEON::BI__builtin_neon_vget_lane_bf16:
10740 case NEON::BI__builtin_neon_vduph_lane_bf16:
10741 case NEON::BI__builtin_neon_vduph_lane_f16: {
10745 case NEON::BI__builtin_neon_vgetq_lane_bf16:
10746 case NEON::BI__builtin_neon_vduph_laneq_bf16:
10747 case NEON::BI__builtin_neon_vduph_laneq_f16: {
10752 case clang::AArch64::BI_InterlockedAdd: {
10755 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
10756 AtomicRMWInst::Add, Arg0, Arg1,
10757 llvm::AtomicOrdering::SequentiallyConsistent);
10758 return Builder.CreateAdd(RMWI, Arg1);
10763 llvm::Type *Ty = VTy;
10774 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
10775 Builtin->NameHint, Builtin->TypeModifier, E, Ops,
10782 switch (BuiltinID) {
10783 default:
return nullptr;
10784 case NEON::BI__builtin_neon_vbsl_v:
10785 case NEON::BI__builtin_neon_vbslq_v: {
10786 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
10787 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy,
"vbsl");
10788 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy,
"vbsl");
10789 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy,
"vbsl");
10791 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1],
"vbsl");
10792 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2],
"vbsl");
10793 Ops[0] = Builder.CreateOr(Ops[1], Ops[2],
"vbsl");
10794 return Builder.CreateBitCast(Ops[0], Ty);
10796 case NEON::BI__builtin_neon_vfma_lane_v:
10797 case NEON::BI__builtin_neon_vfmaq_lane_v: {
10800 Value *Addend = Ops[0];
10801 Value *Multiplicand = Ops[1];
10802 Value *LaneSource = Ops[2];
10803 Ops[0] = Multiplicand;
10804 Ops[1] = LaneSource;
10808 auto *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v
10809 ? llvm::FixedVectorType::get(VTy->getElementType(),
10810 VTy->getNumElements() / 2)
10813 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(), cst);
10814 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy);
10815 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV,
"lane");
10818 Int = Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
10822 case NEON::BI__builtin_neon_vfma_laneq_v: {
10825 if (VTy && VTy->getElementType() ==
DoubleTy) {
10826 Ops[0] = Builder.CreateBitCast(Ops[0],
DoubleTy);
10827 Ops[1] = Builder.CreateBitCast(Ops[1],
DoubleTy);
10828 llvm::FixedVectorType *VTy =
10830 Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
10831 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
10834 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
10835 DoubleTy, {Ops[1], Ops[2], Ops[0]});
10836 return Builder.CreateBitCast(
Result, Ty);
10838 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
10839 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
10841 auto *STy = llvm::FixedVectorType::get(VTy->getElementType(),
10842 VTy->getNumElements() * 2);
10843 Ops[2] = Builder.CreateBitCast(Ops[2], STy);
10844 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(),
10845 cast<ConstantInt>(Ops[3]));
10846 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV,
"lane");
10849 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
10850 {Ops[2], Ops[1], Ops[0]});
10852 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
10853 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
10854 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
10856 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
10859 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
10860 {Ops[2], Ops[1], Ops[0]});
10862 case NEON::BI__builtin_neon_vfmah_lane_f16:
10863 case NEON::BI__builtin_neon_vfmas_lane_f32:
10864 case NEON::BI__builtin_neon_vfmah_laneq_f16:
10865 case NEON::BI__builtin_neon_vfmas_laneq_f32:
10866 case NEON::BI__builtin_neon_vfmad_lane_f64:
10867 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
10870 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
10872 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
10873 {Ops[1], Ops[2], Ops[0]});
10875 case NEON::BI__builtin_neon_vmull_v:
10877 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
10878 if (
Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull;
10880 case NEON::BI__builtin_neon_vmax_v:
10881 case NEON::BI__builtin_neon_vmaxq_v:
10883 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
10884 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
10886 case NEON::BI__builtin_neon_vmaxh_f16: {
10888 Int = Intrinsic::aarch64_neon_fmax;
10891 case NEON::BI__builtin_neon_vmin_v:
10892 case NEON::BI__builtin_neon_vminq_v:
10894 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
10895 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
10897 case NEON::BI__builtin_neon_vminh_f16: {
10899 Int = Intrinsic::aarch64_neon_fmin;
10902 case NEON::BI__builtin_neon_vabd_v:
10903 case NEON::BI__builtin_neon_vabdq_v:
10905 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
10906 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
10908 case NEON::BI__builtin_neon_vpadal_v:
10909 case NEON::BI__builtin_neon_vpadalq_v: {
10910 unsigned ArgElts = VTy->getNumElements();
10912 unsigned BitWidth = EltTy->getBitWidth();
10913 auto *ArgTy = llvm::FixedVectorType::get(
10914 llvm::IntegerType::get(
getLLVMContext(), BitWidth / 2), 2 * ArgElts);
10915 llvm::Type* Tys[2] = { VTy, ArgTy };
10916 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
10918 TmpOps.push_back(Ops[1]);
10921 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType());
10922 return Builder.CreateAdd(tmp, addend);
10924 case NEON::BI__builtin_neon_vpmin_v:
10925 case NEON::BI__builtin_neon_vpminq_v:
10927 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
10928 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
10930 case NEON::BI__builtin_neon_vpmax_v:
10931 case NEON::BI__builtin_neon_vpmaxq_v:
10933 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
10934 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
10936 case NEON::BI__builtin_neon_vminnm_v:
10937 case NEON::BI__builtin_neon_vminnmq_v:
10938 Int = Intrinsic::aarch64_neon_fminnm;
10940 case NEON::BI__builtin_neon_vminnmh_f16:
10942 Int = Intrinsic::aarch64_neon_fminnm;
10944 case NEON::BI__builtin_neon_vmaxnm_v:
10945 case NEON::BI__builtin_neon_vmaxnmq_v:
10946 Int = Intrinsic::aarch64_neon_fmaxnm;
10948 case NEON::BI__builtin_neon_vmaxnmh_f16:
10950 Int = Intrinsic::aarch64_neon_fmaxnm;
10952 case NEON::BI__builtin_neon_vrecpss_f32: {
10957 case NEON::BI__builtin_neon_vrecpsd_f64:
10961 case NEON::BI__builtin_neon_vrecpsh_f16:
10965 case NEON::BI__builtin_neon_vqshrun_n_v:
10966 Int = Intrinsic::aarch64_neon_sqshrun;
10968 case NEON::BI__builtin_neon_vqrshrun_n_v:
10969 Int = Intrinsic::aarch64_neon_sqrshrun;
10971 case NEON::BI__builtin_neon_vqshrn_n_v:
10972 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
10974 case NEON::BI__builtin_neon_vrshrn_n_v:
10975 Int = Intrinsic::aarch64_neon_rshrn;
10977 case NEON::BI__builtin_neon_vqrshrn_n_v:
10978 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
10980 case NEON::BI__builtin_neon_vrndah_f16: {
10982 Int = Builder.getIsFPConstrained()
10983 ? Intrinsic::experimental_constrained_round
10984 : Intrinsic::round;
10987 case NEON::BI__builtin_neon_vrnda_v:
10988 case NEON::BI__builtin_neon_vrndaq_v: {
10989 Int = Builder.getIsFPConstrained()
10990 ? Intrinsic::experimental_constrained_round
10991 : Intrinsic::round;
10994 case NEON::BI__builtin_neon_vrndih_f16: {
10996 Int = Builder.getIsFPConstrained()
10997 ? Intrinsic::experimental_constrained_nearbyint
10998 : Intrinsic::nearbyint;
11001 case NEON::BI__builtin_neon_vrndmh_f16: {
11003 Int = Builder.getIsFPConstrained()
11004 ? Intrinsic::experimental_constrained_floor
11005 : Intrinsic::floor;
11008 case NEON::BI__builtin_neon_vrndm_v:
11009 case NEON::BI__builtin_neon_vrndmq_v: {
11010 Int = Builder.getIsFPConstrained()
11011 ? Intrinsic::experimental_constrained_floor
11012 : Intrinsic::floor;
11015 case NEON::BI__builtin_neon_vrndnh_f16: {
11017 Int = Builder.getIsFPConstrained()
11018 ? Intrinsic::experimental_constrained_roundeven
11019 : Intrinsic::roundeven;
11022 case NEON::BI__builtin_neon_vrndn_v:
11023 case NEON::BI__builtin_neon_vrndnq_v: {
11024 Int = Builder.getIsFPConstrained()
11025 ? Intrinsic::experimental_constrained_roundeven
11026 : Intrinsic::roundeven;
11029 case NEON::BI__builtin_neon_vrndns_f32: {
11031 Int = Builder.getIsFPConstrained()
11032 ? Intrinsic::experimental_constrained_roundeven
11033 : Intrinsic::roundeven;
11036 case NEON::BI__builtin_neon_vrndph_f16: {
11038 Int = Builder.getIsFPConstrained()
11039 ? Intrinsic::experimental_constrained_ceil
11043 case NEON::BI__builtin_neon_vrndp_v:
11044 case NEON::BI__builtin_neon_vrndpq_v: {
11045 Int = Builder.getIsFPConstrained()
11046 ? Intrinsic::experimental_constrained_ceil
11050 case NEON::BI__builtin_neon_vrndxh_f16: {
11052 Int = Builder.getIsFPConstrained()
11053 ? Intrinsic::experimental_constrained_rint
11057 case NEON::BI__builtin_neon_vrndx_v:
11058 case NEON::BI__builtin_neon_vrndxq_v: {
11059 Int = Builder.getIsFPConstrained()
11060 ? Intrinsic::experimental_constrained_rint
11064 case NEON::BI__builtin_neon_vrndh_f16: {
11066 Int = Builder.getIsFPConstrained()
11067 ? Intrinsic::experimental_constrained_trunc
11068 : Intrinsic::trunc;
11071 case NEON::BI__builtin_neon_vrnd32x_v:
11072 case NEON::BI__builtin_neon_vrnd32xq_v: {
11074 Int = Intrinsic::aarch64_neon_frint32x;
11077 case NEON::BI__builtin_neon_vrnd32z_v:
11078 case NEON::BI__builtin_neon_vrnd32zq_v: {
11080 Int = Intrinsic::aarch64_neon_frint32z;
11083 case NEON::BI__builtin_neon_vrnd64x_v:
11084 case NEON::BI__builtin_neon_vrnd64xq_v: {
11086 Int = Intrinsic::aarch64_neon_frint64x;
11089 case NEON::BI__builtin_neon_vrnd64z_v:
11090 case NEON::BI__builtin_neon_vrnd64zq_v: {
11092 Int = Intrinsic::aarch64_neon_frint64z;
11095 case NEON::BI__builtin_neon_vrnd_v:
11096 case NEON::BI__builtin_neon_vrndq_v: {
11097 Int = Builder.getIsFPConstrained()
11098 ? Intrinsic::experimental_constrained_trunc
11099 : Intrinsic::trunc;
11102 case NEON::BI__builtin_neon_vcvt_f64_v:
11103 case NEON::BI__builtin_neon_vcvtq_f64_v:
11104 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
11106 return usgn ? Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
11107 : Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
11108 case NEON::BI__builtin_neon_vcvt_f64_f32: {
11110 "unexpected vcvt_f64_f32 builtin");
11112 Ops[0] = Builder.CreateBitCast(Ops[0],
GetNeonType(
this, SrcFlag));
11114 return Builder.CreateFPExt(Ops[0], Ty,
"vcvt");
11116 case NEON::BI__builtin_neon_vcvt_f32_f64: {
11118 "unexpected vcvt_f32_f64 builtin");
11120 Ops[0] = Builder.CreateBitCast(Ops[0],
GetNeonType(
this, SrcFlag));
11122 return Builder.CreateFPTrunc(Ops[0], Ty,
"vcvt");
11124 case NEON::BI__builtin_neon_vcvt_s32_v:
11125 case NEON::BI__builtin_neon_vcvt_u32_v:
11126 case NEON::BI__builtin_neon_vcvt_s64_v:
11127 case NEON::BI__builtin_neon_vcvt_u64_v:
11128 case NEON::BI__builtin_neon_vcvt_s16_v:
11129 case NEON::BI__builtin_neon_vcvt_u16_v:
11130 case NEON::BI__builtin_neon_vcvtq_s32_v:
11131 case NEON::BI__builtin_neon_vcvtq_u32_v:
11132 case NEON::BI__builtin_neon_vcvtq_s64_v:
11133 case NEON::BI__builtin_neon_vcvtq_u64_v:
11134 case NEON::BI__builtin_neon_vcvtq_s16_v:
11135 case NEON::BI__builtin_neon_vcvtq_u16_v: {
11137 usgn ? Intrinsic::aarch64_neon_fcvtzu : Intrinsic::aarch64_neon_fcvtzs;
11141 case NEON::BI__builtin_neon_vcvta_s16_v:
11142 case NEON::BI__builtin_neon_vcvta_u16_v:
11143 case NEON::BI__builtin_neon_vcvta_s32_v:
11144 case NEON::BI__builtin_neon_vcvtaq_s16_v:
11145 case NEON::BI__builtin_neon_vcvtaq_s32_v:
11146 case NEON::BI__builtin_neon_vcvta_u32_v:
11147 case NEON::BI__builtin_neon_vcvtaq_u16_v:
11148 case NEON::BI__builtin_neon_vcvtaq_u32_v:
11149 case NEON::BI__builtin_neon_vcvta_s64_v:
11150 case NEON::BI__builtin_neon_vcvtaq_s64_v:
11151 case NEON::BI__builtin_neon_vcvta_u64_v:
11152 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
11153 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
11157 case NEON::BI__builtin_neon_vcvtm_s16_v:
11158 case NEON::BI__builtin_neon_vcvtm_s32_v:
11159 case NEON::BI__builtin_neon_vcvtmq_s16_v:
11160 case NEON::BI__builtin_neon_vcvtmq_s32_v:
11161 case NEON::BI__builtin_neon_vcvtm_u16_v:
11162 case NEON::BI__builtin_neon_vcvtm_u32_v:
11163 case NEON::BI__builtin_neon_vcvtmq_u16_v:
11164 case NEON::BI__builtin_neon_vcvtmq_u32_v:
11165 case NEON::BI__builtin_neon_vcvtm_s64_v:
11166 case NEON::BI__builtin_neon_vcvtmq_s64_v:
11167 case NEON::BI__builtin_neon_vcvtm_u64_v:
11168 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
11169 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
11173 case NEON::BI__builtin_neon_vcvtn_s16_v:
11174 case NEON::BI__builtin_neon_vcvtn_s32_v:
11175 case NEON::BI__builtin_neon_vcvtnq_s16_v:
11176 case NEON::BI__builtin_neon_vcvtnq_s32_v:
11177 case NEON::BI__builtin_neon_vcvtn_u16_v:
11178 case NEON::BI__builtin_neon_vcvtn_u32_v:
11179 case NEON::BI__builtin_neon_vcvtnq_u16_v:
11180 case NEON::BI__builtin_neon_vcvtnq_u32_v:
11181 case NEON::BI__builtin_neon_vcvtn_s64_v:
11182 case NEON::BI__builtin_neon_vcvtnq_s64_v:
11183 case NEON::BI__builtin_neon_vcvtn_u64_v:
11184 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
11185 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
11189 case NEON::BI__builtin_neon_vcvtp_s16_v:
11190 case NEON::BI__builtin_neon_vcvtp_s32_v:
11191 case NEON::BI__builtin_neon_vcvtpq_s16_v:
11192 case NEON::BI__builtin_neon_vcvtpq_s32_v:
11193 case NEON::BI__builtin_neon_vcvtp_u16_v:
11194 case NEON::BI__builtin_neon_vcvtp_u32_v:
11195 case NEON::BI__builtin_neon_vcvtpq_u16_v:
11196 case NEON::BI__builtin_neon_vcvtpq_u32_v:
11197 case NEON::BI__builtin_neon_vcvtp_s64_v:
11198 case NEON::BI__builtin_neon_vcvtpq_s64_v:
11199 case NEON::BI__builtin_neon_vcvtp_u64_v:
11200 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
11201 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
11205 case NEON::BI__builtin_neon_vmulx_v:
11206 case NEON::BI__builtin_neon_vmulxq_v: {
11207 Int = Intrinsic::aarch64_neon_fmulx;
11210 case NEON::BI__builtin_neon_vmulxh_lane_f16:
11211 case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
11215 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
11217 Int = Intrinsic::aarch64_neon_fmulx;
11220 case NEON::BI__builtin_neon_vmul_lane_v:
11221 case NEON::BI__builtin_neon_vmul_laneq_v: {
11224 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
11226 Ops[0] = Builder.CreateBitCast(Ops[0],
DoubleTy);
11227 llvm::FixedVectorType *VTy =
11229 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
11230 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
11231 Value *
Result = Builder.CreateFMul(Ops[0], Ops[1]);
11232 return Builder.CreateBitCast(
Result, Ty);
11234 case NEON::BI__builtin_neon_vnegd_s64:
11236 case NEON::BI__builtin_neon_vnegh_f16:
11238 case NEON::BI__builtin_neon_vpmaxnm_v:
11239 case NEON::BI__builtin_neon_vpmaxnmq_v: {
11240 Int = Intrinsic::aarch64_neon_fmaxnmp;
11243 case NEON::BI__builtin_neon_vpminnm_v:
11244 case NEON::BI__builtin_neon_vpminnmq_v: {
11245 Int = Intrinsic::aarch64_neon_fminnmp;
11248 case NEON::BI__builtin_neon_vsqrth_f16: {
11250 Int = Builder.getIsFPConstrained()
11251 ? Intrinsic::experimental_constrained_sqrt
11255 case NEON::BI__builtin_neon_vsqrt_v:
11256 case NEON::BI__builtin_neon_vsqrtq_v: {
11257 Int = Builder.getIsFPConstrained()
11258 ? Intrinsic::experimental_constrained_sqrt
11260 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
11263 case NEON::BI__builtin_neon_vrbit_v:
11264 case NEON::BI__builtin_neon_vrbitq_v: {
11265 Int = Intrinsic::bitreverse;
11268 case NEON::BI__builtin_neon_vaddv_u8:
11272 case NEON::BI__builtin_neon_vaddv_s8: {
11273 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
11275 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
11276 llvm::Type *Tys[2] = { Ty, VTy };
11279 return Builder.CreateTrunc(Ops[0],
Int8Ty);
11281 case NEON::BI__builtin_neon_vaddv_u16:
11284 case NEON::BI__builtin_neon_vaddv_s16: {
11285 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
11287 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
11288 llvm::Type *Tys[2] = { Ty, VTy };
11291 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11293 case NEON::BI__builtin_neon_vaddvq_u8:
11296 case NEON::BI__builtin_neon_vaddvq_s8: {
11297 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
11299 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
11300 llvm::Type *Tys[2] = { Ty, VTy };
11303 return Builder.CreateTrunc(Ops[0],
Int8Ty);
11305 case NEON::BI__builtin_neon_vaddvq_u16:
11308 case NEON::BI__builtin_neon_vaddvq_s16: {
11309 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
11311 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
11312 llvm::Type *Tys[2] = { Ty, VTy };
11315 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11317 case NEON::BI__builtin_neon_vmaxv_u8: {
11318 Int = Intrinsic::aarch64_neon_umaxv;
11320 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
11321 llvm::Type *Tys[2] = { Ty, VTy };
11324 return Builder.CreateTrunc(Ops[0],
Int8Ty);
11326 case NEON::BI__builtin_neon_vmaxv_u16: {
11327 Int = Intrinsic::aarch64_neon_umaxv;
11329 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
11330 llvm::Type *Tys[2] = { Ty, VTy };
11333 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11335 case NEON::BI__builtin_neon_vmaxvq_u8: {
11336 Int = Intrinsic::aarch64_neon_umaxv;
11338 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
11339 llvm::Type *Tys[2] = { Ty, VTy };
11342 return Builder.CreateTrunc(Ops[0],
Int8Ty);
11344 case NEON::BI__builtin_neon_vmaxvq_u16: {
11345 Int = Intrinsic::aarch64_neon_umaxv;
11347 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
11348 llvm::Type *Tys[2] = { Ty, VTy };
11351 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11353 case NEON::BI__builtin_neon_vmaxv_s8: {
11354 Int = Intrinsic::aarch64_neon_smaxv;
11356 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
11357 llvm::Type *Tys[2] = { Ty, VTy };
11360 return Builder.CreateTrunc(Ops[0],
Int8Ty);
11362 case NEON::BI__builtin_neon_vmaxv_s16: {
11363 Int = Intrinsic::aarch64_neon_smaxv;
11365 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
11366 llvm::Type *Tys[2] = { Ty, VTy };
11369 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11371 case NEON::BI__builtin_neon_vmaxvq_s8: {
11372 Int = Intrinsic::aarch64_neon_smaxv;
11374 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
11375 llvm::Type *Tys[2] = { Ty, VTy };
11378 return Builder.CreateTrunc(Ops[0],
Int8Ty);
11380 case NEON::BI__builtin_neon_vmaxvq_s16: {
11381 Int = Intrinsic::aarch64_neon_smaxv;
11383 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
11384 llvm::Type *Tys[2] = { Ty, VTy };
11387 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11389 case NEON::BI__builtin_neon_vmaxv_f16: {
11390 Int = Intrinsic::aarch64_neon_fmaxv;
11392 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
11393 llvm::Type *Tys[2] = { Ty, VTy };
11396 return Builder.CreateTrunc(Ops[0],
HalfTy);
11398 case NEON::BI__builtin_neon_vmaxvq_f16: {
11399 Int = Intrinsic::aarch64_neon_fmaxv;
11401 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
11402 llvm::Type *Tys[2] = { Ty, VTy };
11405 return Builder.CreateTrunc(Ops[0],
HalfTy);
11407 case NEON::BI__builtin_neon_vminv_u8: {
11408 Int = Intrinsic::aarch64_neon_uminv;
11410 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
11411 llvm::Type *Tys[2] = { Ty, VTy };
11414 return Builder.CreateTrunc(Ops[0],
Int8Ty);
11416 case NEON::BI__builtin_neon_vminv_u16: {
11417 Int = Intrinsic::aarch64_neon_uminv;
11419 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
11420 llvm::Type *Tys[2] = { Ty, VTy };
11423 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11425 case NEON::BI__builtin_neon_vminvq_u8: {
11426 Int = Intrinsic::aarch64_neon_uminv;
11428 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
11429 llvm::Type *Tys[2] = { Ty, VTy };
11432 return Builder.CreateTrunc(Ops[0],
Int8Ty);
11434 case NEON::BI__builtin_neon_vminvq_u16: {
11435 Int = Intrinsic::aarch64_neon_uminv;
11437 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
11438 llvm::Type *Tys[2] = { Ty, VTy };
11441 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11443 case NEON::BI__builtin_neon_vminv_s8: {
11444 Int = Intrinsic::aarch64_neon_sminv;
11446 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
11447 llvm::Type *Tys[2] = { Ty, VTy };
11450 return Builder.CreateTrunc(Ops[0],
Int8Ty);
11452 case NEON::BI__builtin_neon_vminv_s16: {
11453 Int = Intrinsic::aarch64_neon_sminv;
11455 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
11456 llvm::Type *Tys[2] = { Ty, VTy };
11459 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11461 case NEON::BI__builtin_neon_vminvq_s8: {
11462 Int = Intrinsic::aarch64_neon_sminv;
11464 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
11465 llvm::Type *Tys[2] = { Ty, VTy };
11468 return Builder.CreateTrunc(Ops[0],
Int8Ty);
11470 case NEON::BI__builtin_neon_vminvq_s16: {
11471 Int = Intrinsic::aarch64_neon_sminv;
11473 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
11474 llvm::Type *Tys[2] = { Ty, VTy };
11477 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11479 case NEON::BI__builtin_neon_vminv_f16: {
11480 Int = Intrinsic::aarch64_neon_fminv;
11482 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
11483 llvm::Type *Tys[2] = { Ty, VTy };
11486 return Builder.CreateTrunc(Ops[0],
HalfTy);
11488 case NEON::BI__builtin_neon_vminvq_f16: {
11489 Int = Intrinsic::aarch64_neon_fminv;
11491 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
11492 llvm::Type *Tys[2] = { Ty, VTy };
11495 return Builder.CreateTrunc(Ops[0],
HalfTy);
11497 case NEON::BI__builtin_neon_vmaxnmv_f16: {
11498 Int = Intrinsic::aarch64_neon_fmaxnmv;
11500 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
11501 llvm::Type *Tys[2] = { Ty, VTy };
11504 return Builder.CreateTrunc(Ops[0],
HalfTy);
11506 case NEON::BI__builtin_neon_vmaxnmvq_f16: {
11507 Int = Intrinsic::aarch64_neon_fmaxnmv;
11509 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
11510 llvm::Type *Tys[2] = { Ty, VTy };
11513 return Builder.CreateTrunc(Ops[0],
HalfTy);
11515 case NEON::BI__builtin_neon_vminnmv_f16: {
11516 Int = Intrinsic::aarch64_neon_fminnmv;
11518 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
11519 llvm::Type *Tys[2] = { Ty, VTy };
11522 return Builder.CreateTrunc(Ops[0],
HalfTy);
11524 case NEON::BI__builtin_neon_vminnmvq_f16: {
11525 Int = Intrinsic::aarch64_neon_fminnmv;
11527 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
11528 llvm::Type *Tys[2] = { Ty, VTy };
11531 return Builder.CreateTrunc(Ops[0],
HalfTy);
11533 case NEON::BI__builtin_neon_vmul_n_f64: {
11534 Ops[0] = Builder.CreateBitCast(Ops[0],
DoubleTy);
11536 return Builder.CreateFMul(Ops[0], RHS);
11538 case NEON::BI__builtin_neon_vaddlv_u8: {
11539 Int = Intrinsic::aarch64_neon_uaddlv;
11541 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
11542 llvm::Type *Tys[2] = { Ty, VTy };
11545 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11547 case NEON::BI__builtin_neon_vaddlv_u16: {
11548 Int = Intrinsic::aarch64_neon_uaddlv;
11550 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
11551 llvm::Type *Tys[2] = { Ty, VTy };
11555 case NEON::BI__builtin_neon_vaddlvq_u8: {
11556 Int = Intrinsic::aarch64_neon_uaddlv;
11558 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
11559 llvm::Type *Tys[2] = { Ty, VTy };
11562 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11564 case NEON::BI__builtin_neon_vaddlvq_u16: {
11565 Int = Intrinsic::aarch64_neon_uaddlv;
11567 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
11568 llvm::Type *Tys[2] = { Ty, VTy };
11572 case NEON::BI__builtin_neon_vaddlv_s8: {
11573 Int = Intrinsic::aarch64_neon_saddlv;
11575 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
11576 llvm::Type *Tys[2] = { Ty, VTy };
11579 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11581 case NEON::BI__builtin_neon_vaddlv_s16: {
11582 Int = Intrinsic::aarch64_neon_saddlv;
11584 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
11585 llvm::Type *Tys[2] = { Ty, VTy };
11589 case NEON::BI__builtin_neon_vaddlvq_s8: {
11590 Int = Intrinsic::aarch64_neon_saddlv;
11592 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
11593 llvm::Type *Tys[2] = { Ty, VTy };
11596 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11598 case NEON::BI__builtin_neon_vaddlvq_s16: {
11599 Int = Intrinsic::aarch64_neon_saddlv;
11601 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
11602 llvm::Type *Tys[2] = { Ty, VTy };
11606 case NEON::BI__builtin_neon_vsri_n_v:
11607 case NEON::BI__builtin_neon_vsriq_n_v: {
11608 Int = Intrinsic::aarch64_neon_vsri;
11612 case NEON::BI__builtin_neon_vsli_n_v:
11613 case NEON::BI__builtin_neon_vsliq_n_v: {
11614 Int = Intrinsic::aarch64_neon_vsli;
11618 case NEON::BI__builtin_neon_vsra_n_v:
11619 case NEON::BI__builtin_neon_vsraq_n_v:
11620 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
11622 return Builder.CreateAdd(Ops[0], Ops[1]);
11623 case NEON::BI__builtin_neon_vrsra_n_v:
11624 case NEON::BI__builtin_neon_vrsraq_n_v: {
11625 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
11627 TmpOps.push_back(Ops[1]);
11628 TmpOps.push_back(Ops[2]);
11630 llvm::Value *tmp =
EmitNeonCall(F, TmpOps,
"vrshr_n", 1,
true);
11631 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
11632 return Builder.CreateAdd(Ops[0], tmp);
11634 case NEON::BI__builtin_neon_vld1_v:
11635 case NEON::BI__builtin_neon_vld1q_v: {
11636 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
11637 return Builder.CreateAlignedLoad(VTy, Ops[0], PtrOp0.
getAlignment());
11639 case NEON::BI__builtin_neon_vst1_v:
11640 case NEON::BI__builtin_neon_vst1q_v:
11641 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
11642 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
11643 return Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.
getAlignment());
11644 case NEON::BI__builtin_neon_vld1_lane_v:
11645 case NEON::BI__builtin_neon_vld1q_lane_v: {
11646 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
11647 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
11648 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
11649 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
11651 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vld1_lane");
11653 case NEON::BI__builtin_neon_vld1_dup_v:
11654 case NEON::BI__builtin_neon_vld1q_dup_v: {
11655 Value *
V = UndefValue::get(Ty);
11656 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
11657 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
11658 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
11660 llvm::Constant *CI = ConstantInt::get(
Int32Ty, 0);
11661 Ops[0] = Builder.CreateInsertElement(
V, Ops[0], CI);
11664 case NEON::BI__builtin_neon_vst1_lane_v:
11665 case NEON::BI__builtin_neon_vst1q_lane_v:
11666 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
11667 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
11668 Ty = llvm::PointerType::getUnqual(Ops[1]->
getType());
11669 return Builder.CreateAlignedStore(Ops[1], Builder.CreateBitCast(Ops[0], Ty),
11671 case NEON::BI__builtin_neon_vld2_v:
11672 case NEON::BI__builtin_neon_vld2q_v: {
11673 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
11674 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
11675 llvm::Type *Tys[2] = { VTy, PTy };
11677 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld2");
11678 Ops[0] = Builder.CreateBitCast(Ops[0],
11679 llvm::PointerType::getUnqual(Ops[1]->
getType()));
11680 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
11682 case NEON::BI__builtin_neon_vld3_v:
11683 case NEON::BI__builtin_neon_vld3q_v: {
11684 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
11685 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
11686 llvm::Type *Tys[2] = { VTy, PTy };
11688 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld3");
11689 Ops[0] = Builder.CreateBitCast(Ops[0],
11690 llvm::PointerType::getUnqual(Ops[1]->
getType()));
11691 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
11693 case NEON::BI__builtin_neon_vld4_v:
11694 case NEON::BI__builtin_neon_vld4q_v: {
11695 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
11696 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
11697 llvm::Type *Tys[2] = { VTy, PTy };
11699 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld4");
11700 Ops[0] = Builder.CreateBitCast(Ops[0],
11701 llvm::PointerType::getUnqual(Ops[1]->
getType()));
11702 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
11704 case NEON::BI__builtin_neon_vld2_dup_v:
11705 case NEON::BI__builtin_neon_vld2q_dup_v: {
11707 llvm::PointerType::getUnqual(VTy->getElementType());
11708 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
11709 llvm::Type *Tys[2] = { VTy, PTy };
11711 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld2");
11712 Ops[0] = Builder.CreateBitCast(Ops[0],
11713 llvm::PointerType::getUnqual(Ops[1]->
getType()));
11714 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
11716 case NEON::BI__builtin_neon_vld3_dup_v:
11717 case NEON::BI__builtin_neon_vld3q_dup_v: {
11719 llvm::PointerType::getUnqual(VTy->getElementType());
11720 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
11721 llvm::Type *Tys[2] = { VTy, PTy };
11723 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld3");
11724 Ops[0] = Builder.CreateBitCast(Ops[0],
11725 llvm::PointerType::getUnqual(Ops[1]->
getType()));
11726 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
11728 case NEON::BI__builtin_neon_vld4_dup_v:
11729 case NEON::BI__builtin_neon_vld4q_dup_v: {
11731 llvm::PointerType::getUnqual(VTy->getElementType());
11732 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
11733 llvm::Type *Tys[2] = { VTy, PTy };
11735 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld4");
11736 Ops[0] = Builder.CreateBitCast(Ops[0],
11737 llvm::PointerType::getUnqual(Ops[1]->
getType()));
11738 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
11740 case NEON::BI__builtin_neon_vld2_lane_v:
11741 case NEON::BI__builtin_neon_vld2q_lane_v: {
11742 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
11743 Function *F =
CGM.
getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys);
11744 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
11745 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
11746 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
11747 Ops[3] = Builder.CreateZExt(Ops[3],
Int64Ty);
11748 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1),
"vld2_lane");
11749 Ty = llvm::PointerType::getUnqual(Ops[1]->
getType());
11750 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
11751 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
11753 case NEON::BI__builtin_neon_vld3_lane_v:
11754 case NEON::BI__builtin_neon_vld3q_lane_v: {
11755 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
11756 Function *F =
CGM.
getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys);
11757 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
11758 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
11759 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
11760 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
11761 Ops[4] = Builder.CreateZExt(Ops[4],
Int64Ty);
11762 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1),
"vld3_lane");
11763 Ty = llvm::PointerType::getUnqual(Ops[1]->
getType());
11764 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
11765 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
11767 case NEON::BI__builtin_neon_vld4_lane_v:
11768 case NEON::BI__builtin_neon_vld4q_lane_v: {
11769 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
11770 Function *F =
CGM.
getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys);
11771 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
11772 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
11773 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
11774 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
11775 Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
11776 Ops[5] = Builder.CreateZExt(Ops[5],
Int64Ty);
11777 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1),
"vld4_lane");
11778 Ty = llvm::PointerType::getUnqual(Ops[1]->
getType());
11779 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
11780 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
11782 case NEON::BI__builtin_neon_vst2_v:
11783 case NEON::BI__builtin_neon_vst2q_v: {
11784 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
11785 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
11789 case NEON::BI__builtin_neon_vst2_lane_v:
11790 case NEON::BI__builtin_neon_vst2q_lane_v: {
11791 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
11792 Ops[2] = Builder.CreateZExt(Ops[2],
Int64Ty);
11793 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
11797 case NEON::BI__builtin_neon_vst3_v:
11798 case NEON::BI__builtin_neon_vst3q_v: {
11799 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
11800 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
11804 case NEON::BI__builtin_neon_vst3_lane_v:
11805 case NEON::BI__builtin_neon_vst3q_lane_v: {
11806 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
11807 Ops[3] = Builder.CreateZExt(Ops[3],
Int64Ty);
11808 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
11812 case NEON::BI__builtin_neon_vst4_v:
11813 case NEON::BI__builtin_neon_vst4q_v: {
11814 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
11815 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
11819 case NEON::BI__builtin_neon_vst4_lane_v:
11820 case NEON::BI__builtin_neon_vst4q_lane_v: {
11821 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
11822 Ops[4] = Builder.CreateZExt(Ops[4],
Int64Ty);
11823 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
11827 case NEON::BI__builtin_neon_vtrn_v:
11828 case NEON::BI__builtin_neon_vtrnq_v: {
11829 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
11830 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
11831 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
11832 Value *SV =
nullptr;
11834 for (
unsigned vi = 0; vi != 2; ++vi) {
11836 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
11837 Indices.push_back(i+vi);
11838 Indices.push_back(i+e+vi);
11840 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
11841 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
11842 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
11846 case NEON::BI__builtin_neon_vuzp_v:
11847 case NEON::BI__builtin_neon_vuzpq_v: {
11848 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
11849 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
11850 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
11851 Value *SV =
nullptr;
11853 for (
unsigned vi = 0; vi != 2; ++vi) {
11855 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
11856 Indices.push_back(2*i+vi);
11858 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
11859 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
11860 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
11864 case NEON::BI__builtin_neon_vzip_v:
11865 case NEON::BI__builtin_neon_vzipq_v: {
11866 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
11867 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
11868 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
11869 Value *SV =
nullptr;
11871 for (
unsigned vi = 0; vi != 2; ++vi) {
11873 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
11874 Indices.push_back((i + vi*e) >> 1);
11875 Indices.push_back(((i + vi*e) >> 1)+e);
11877 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
11878 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
11879 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
11883 case NEON::BI__builtin_neon_vqtbl1q_v: {
11887 case NEON::BI__builtin_neon_vqtbl2q_v: {
11891 case NEON::BI__builtin_neon_vqtbl3q_v: {
11895 case NEON::BI__builtin_neon_vqtbl4q_v: {
11899 case NEON::BI__builtin_neon_vqtbx1q_v: {
11903 case NEON::BI__builtin_neon_vqtbx2q_v: {
11907 case NEON::BI__builtin_neon_vqtbx3q_v: {
11911 case NEON::BI__builtin_neon_vqtbx4q_v: {
11915 case NEON::BI__builtin_neon_vsqadd_v:
11916 case NEON::BI__builtin_neon_vsqaddq_v: {
11917 Int = Intrinsic::aarch64_neon_usqadd;
11920 case NEON::BI__builtin_neon_vuqadd_v:
11921 case NEON::BI__builtin_neon_vuqaddq_v: {
11922 Int = Intrinsic::aarch64_neon_suqadd;
11930 assert((BuiltinID == BPF::BI__builtin_preserve_field_info ||
11931 BuiltinID == BPF::BI__builtin_btf_type_id ||
11932 BuiltinID == BPF::BI__builtin_preserve_type_info ||
11933 BuiltinID == BPF::BI__builtin_preserve_enum_value) &&
11934 "unexpected BPF builtin");
11939 static uint32_t BuiltinSeqNum;
11941 switch (BuiltinID) {
11943 llvm_unreachable(
"Unexpected BPF builtin");
11944 case BPF::BI__builtin_preserve_field_info: {
11950 "using __builtin_preserve_field_info() without -g");
11963 Value *InfoKind = ConstantInt::get(
Int64Ty,
C->getSExtValue());
11966 llvm::Function *FnGetFieldInfo = llvm::Intrinsic::getDeclaration(
11967 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_field_info,
11968 {FieldAddr->getType()});
11969 return Builder.CreateCall(FnGetFieldInfo, {FieldAddr, InfoKind});
11971 case BPF::BI__builtin_btf_type_id:
11972 case BPF::BI__builtin_preserve_type_info: {
11983 Value *FlagValue = ConstantInt::get(
Int64Ty, Flag->getSExtValue());
11984 Value *SeqNumVal = ConstantInt::get(
Int32Ty, BuiltinSeqNum++);
11986 llvm::Function *FnDecl;
11987 if (BuiltinID == BPF::BI__builtin_btf_type_id)
11988 FnDecl = llvm::Intrinsic::getDeclaration(
11989 &
CGM.
getModule(), llvm::Intrinsic::bpf_btf_type_id, {});
11991 FnDecl = llvm::Intrinsic::getDeclaration(
11992 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_type_info, {});
11993 CallInst *Fn = Builder.CreateCall(FnDecl, {SeqNumVal, FlagValue});
11994 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
11997 case BPF::BI__builtin_preserve_enum_value: {
12013 auto &InitVal = Enumerator->getInitVal();
12014 std::string InitValStr;
12015 if (InitVal.isNegative() || InitVal >
uint64_t(INT64_MAX))
12016 InitValStr = std::to_string(InitVal.getSExtValue());
12018 InitValStr = std::to_string(InitVal.getZExtValue());
12019 std::string EnumStr = Enumerator->getNameAsString() +
":" + InitValStr;
12020 Value *EnumStrVal = Builder.CreateGlobalStringPtr(EnumStr);
12023 Value *FlagValue = ConstantInt::get(
Int64Ty, Flag->getSExtValue());
12024 Value *SeqNumVal = ConstantInt::get(
Int32Ty, BuiltinSeqNum++);
12026 llvm::Function *IntrinsicFn = llvm::Intrinsic::getDeclaration(
12027 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_enum_value, {});
12029 Builder.CreateCall(IntrinsicFn, {SeqNumVal, EnumStrVal, FlagValue});
12030 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
12038 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
12039 "Not a power-of-two sized vector!");
12040 bool AllConstants =
true;
12041 for (
unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
12045 if (AllConstants) {
12047 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
12048 CstOps.push_back(cast<Constant>(Ops[i]));
12049 return llvm::ConstantVector::get(CstOps);
12054 llvm::FixedVectorType::get(Ops[0]->
getType(), Ops.size()));
12056 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
12057 Result = Builder.CreateInsertElement(
Result, Ops[i], Builder.getInt32(i));
12064 unsigned NumElts) {
12066 auto *MaskTy = llvm::FixedVectorType::get(
12068 cast<IntegerType>(Mask->getType())->getBitWidth());
12069 Value *MaskVec = CGF.
Builder.CreateBitCast(Mask, MaskTy);
12075 for (
unsigned i = 0; i != NumElts; ++i)
12077 MaskVec = CGF.
Builder.CreateShuffleVector(MaskVec, MaskVec,
12078 makeArrayRef(Indices, NumElts),
12088 llvm::PointerType::getUnqual(Ops[1]->
getType()));
12092 cast<llvm::FixedVectorType>(Ops[1]->
getType())->getNumElements());
12094 return CGF.
Builder.CreateMaskedStore(Ops[1], Ptr, Alignment, MaskVec);
12100 llvm::Type *Ty = Ops[1]->getType();
12102 CGF.
Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
12105 CGF, Ops[2], cast<llvm::FixedVectorType>(Ty)->getNumElements());
12107 return CGF.
Builder.CreateMaskedLoad(Ty, Ptr, Alignment, MaskVec, Ops[1]);
12113 llvm::Type *PtrTy = ResultTy->getElementType();
12117 llvm::PointerType::getUnqual(PtrTy));
12120 CGF, Ops[2], cast<FixedVectorType>(ResultTy)->getNumElements());
12122 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_expandload,
12124 return CGF.
Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] });
12134 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
12135 : Intrinsic::x86_avx512_mask_expand;
12137 return CGF.
Builder.CreateCall(F, { Ops[0], Ops[1], MaskVec });
12143 llvm::Type *PtrTy = ResultTy->getElementType();
12147 llvm::PointerType::getUnqual(PtrTy));
12151 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_compressstore,
12153 return CGF.
Builder.CreateCall(F, { Ops[1], Ptr, MaskVec });
12158 bool InvertLHS =
false) {
12159 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
12164 LHS = CGF.
Builder.CreateNot(LHS);
12166 return CGF.
Builder.CreateBitCast(CGF.
Builder.CreateBinOp(Opc, LHS, RHS),
12167 Ops[0]->getType());
12171 Value *Amt,
bool IsRight) {
12172 llvm::Type *Ty = Op0->getType();
12177 if (Amt->getType() != Ty) {
12179 Amt = CGF.
Builder.CreateIntCast(Amt, Ty->getScalarType(),
false);
12180 Amt = CGF.
Builder.CreateVectorSplat(NumElts, Amt);
12183 unsigned IID = IsRight ? Intrinsic::fshr : Intrinsic::fshl;
12185 return CGF.
Builder.CreateCall(F, {Op0, Op1, Amt});
12190 Value *Op0 = Ops[0];
12191 Value *Op1 = Ops[1];
12192 llvm::Type *Ty = Op0->getType();
12195 CmpInst::Predicate Pred;
12198 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
12201 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
12204 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
12207 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
12210 Pred = ICmpInst::ICMP_EQ;
12213 Pred = ICmpInst::ICMP_NE;
12216 return llvm::Constant::getNullValue(Ty);
12218 return llvm::Constant::getAllOnesValue(Ty);
12220 llvm_unreachable(
"Unexpected XOP vpcom/vpcomu predicate");
12232 if (
const auto *
C = dyn_cast<Constant>(Mask))
12233 if (
C->isAllOnesValue())
12237 CGF, Mask, cast<llvm::FixedVectorType>(Op0->getType())->getNumElements());
12239 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
12245 if (
const auto *
C = dyn_cast<Constant>(Mask))
12246 if (
C->isAllOnesValue())
12249 auto *MaskTy = llvm::FixedVectorType::get(
12250 CGF.
Builder.getInt1Ty(), Mask->getType()->getIntegerBitWidth());
12251 Mask = CGF.
Builder.CreateBitCast(Mask, MaskTy);
12253 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
12257 unsigned NumElts,
Value *MaskIn) {
12259 const auto *
C = dyn_cast<Constant>(MaskIn);
12260 if (!
C || !
C->isAllOnesValue())
12266 for (
unsigned i = 0; i != NumElts; ++i)
12268 for (
unsigned i = NumElts; i != 8; ++i)
12269 Indices[i] = i % NumElts + NumElts;
12270 Cmp = CGF.
Builder.CreateShuffleVector(
12271 Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices);
12274 return CGF.
Builder.CreateBitCast(Cmp,
12276 std::max(NumElts, 8U)));
12281 assert((Ops.size() == 2 || Ops.size() == 4) &&
12282 "Unexpected number of arguments");
12288 Cmp = Constant::getNullValue(
12289 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
12290 }
else if (CC == 7) {
12291 Cmp = Constant::getAllOnesValue(
12292 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
12294 ICmpInst::Predicate Pred;
12296 default: llvm_unreachable(
"Unknown condition code");
12297 case 0: Pred = ICmpInst::ICMP_EQ;
break;
12298 case 1: Pred =
Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
break;
12299 case 2: Pred =
Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
break;
12300 case 4: Pred = ICmpInst::ICMP_NE;
break;
12301 case 5: Pred =
Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
break;
12302 case 6: Pred =
Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
break;
12304 Cmp = CGF.
Builder.CreateICmp(Pred, Ops[0], Ops[1]);
12307 Value *MaskIn =
nullptr;
12308 if (Ops.size() == 4)
12315 Value *Zero = Constant::getNullValue(In->getType());
12322 llvm::Type *Ty = Ops[1]->getType();
12326 Intrinsic::ID IID = IsSigned ? Intrinsic::x86_avx512_sitofp_round
12327 : Intrinsic::x86_avx512_uitofp_round;
12329 Res = CGF.
Builder.CreateCall(F, { Ops[0], Ops[3] });
12331 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
12332 Res = IsSigned ? CGF.
Builder.CreateSIToFP(Ops[0], Ty)
12333 : CGF.
Builder.CreateUIToFP(Ops[0], Ty);
12344 bool Subtract =
false;
12345 Intrinsic::ID IID = Intrinsic::not_intrinsic;
12346 switch (BuiltinID) {
12348 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
12351 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
12352 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
12353 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
12354 IID = llvm::Intrinsic::x86_avx512fp16_vfmadd_ph_512;
12356 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
12359 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
12360 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
12361 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
12362 IID = llvm::Intrinsic::x86_avx512fp16_vfmaddsub_ph_512;
12364 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
12367 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
12368 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
12369 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
12370 IID = llvm::Intrinsic::x86_avx512_vfmadd_ps_512;
break;
12371 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
12374 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
12375 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
12376 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
12377 IID = llvm::Intrinsic::x86_avx512_vfmadd_pd_512;
break;
12378 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
12381 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
12382 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
12383 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
12384 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_ps_512;
12386 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
12389 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
12390 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
12391 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
12392 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_pd_512;
12406 if (IID != Intrinsic::not_intrinsic &&
12407 (cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (
uint64_t)4 ||
12410 Res = CGF.
Builder.CreateCall(Intr, {A, B,
C, Ops.back() });
12412 llvm::Type *Ty = A->getType();
12414 if (CGF.
Builder.getIsFPConstrained()) {
12415 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
12416 FMA = CGF.
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, Ty);
12417 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, {A, B,
C});
12420 Res = CGF.
Builder.CreateCall(FMA, {A, B,
C});
12425 Value *MaskFalseVal =
nullptr;
12426 switch (BuiltinID) {
12427 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
12428 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
12429 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
12430 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
12431 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
12432 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
12433 MaskFalseVal = Ops[0];
12435 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
12436 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
12437 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
12438 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
12439 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
12440 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
12441 MaskFalseVal = Constant::getNullValue(Ops[0]->
getType());
12443 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
12444 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
12445 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
12446 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
12447 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
12448 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
12449 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
12450 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
12451 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
12452 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
12453 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
12454 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
12455 MaskFalseVal = Ops[2];
12467 bool ZeroMask =
false,
unsigned PTIdx = 0,
12468 bool NegAcc =
false) {
12470 if (Ops.size() > 4)
12474 Ops[2] = CGF.
Builder.CreateFNeg(Ops[2]);
12483 switch (Ops[0]->
getType()->getPrimitiveSizeInBits()) {
12485 IID = Intrinsic::x86_avx512fp16_vfmadd_f16;
12488 IID = Intrinsic::x86_avx512_vfmadd_f32;
12491 IID = Intrinsic::x86_avx512_vfmadd_f64;
12494 llvm_unreachable(
"Unexpected size");
12497 {Ops[0], Ops[1], Ops[2], Ops[4]});
12498 }
else if (CGF.
Builder.getIsFPConstrained()) {
12499 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
12501 Intrinsic::experimental_constrained_fma, Ops[0]->
getType());
12502 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, Ops.slice(0, 3));
12505 Res = CGF.
Builder.CreateCall(FMA, Ops.slice(0, 3));
12508 if (Ops.size() > 3) {
12509 Value *PassThru = ZeroMask ? Constant::getNullValue(Res->getType())
12515 if (NegAcc && PTIdx == 2)
12525 llvm::Type *Ty = Ops[0]->getType();
12527 Ty = llvm::FixedVectorType::get(CGF.
Int64Ty,
12528 Ty->getPrimitiveSizeInBits() / 64);
12534 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
12535 LHS = CGF.
Builder.CreateShl(LHS, ShiftAmt);
12536 LHS = CGF.
Builder.CreateAShr(LHS, ShiftAmt);
12537 RHS = CGF.
Builder.CreateShl(RHS, ShiftAmt);
12538 RHS = CGF.
Builder.CreateAShr(RHS, ShiftAmt);
12541 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
12542 LHS = CGF.
Builder.CreateAnd(LHS, Mask);
12543 RHS = CGF.
Builder.CreateAnd(RHS, Mask);
12546 return CGF.
Builder.CreateMul(LHS, RHS);
12554 llvm::Type *Ty = Ops[0]->getType();
12556 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
12557 unsigned EltWidth = Ty->getScalarSizeInBits();
12559 if (VecWidth == 128 && EltWidth == 32)
12560 IID = Intrinsic::x86_avx512_pternlog_d_128;
12561 else if (VecWidth == 256 && EltWidth == 32)
12562 IID = Intrinsic::x86_avx512_pternlog_d_256;
12563 else if (VecWidth == 512 && EltWidth == 32)
12564 IID = Intrinsic::x86_avx512_pternlog_d_512;
12565 else if (VecWidth == 128 && EltWidth == 64)
12566 IID = Intrinsic::x86_avx512_pternlog_q_128;
12567 else if (VecWidth == 256 && EltWidth == 64)
12568 IID = Intrinsic::x86_avx512_pternlog_q_256;
12569 else if (VecWidth == 512 && EltWidth == 64)
12570 IID = Intrinsic::x86_avx512_pternlog_q_512;
12572 llvm_unreachable(
"Unexpected intrinsic");
12576 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) : Ops[0];
12581 llvm::Type *DstTy) {
12582 unsigned NumberOfElements =
12585 return CGF.
Builder.CreateSExt(Mask, DstTy,
"vpmovm2");
12591 return EmitX86CpuIs(CPUStr);
12597 llvm::Type *DstTy) {
12598 assert((Ops.size() == 1 || Ops.size() == 3 || Ops.size() == 4) &&
12599 "Unknown cvtph2ps intrinsic");
12602 if (Ops.size() == 4 && cast<llvm::ConstantInt>(Ops[3])->getZExtValue() != 4) {
12605 return CGF.
Builder.CreateCall(F, {Ops[0], Ops[1], Ops[2], Ops[3]});
12609 Value *Src = Ops[0];
12613 cast<llvm::FixedVectorType>(Src->getType())->getNumElements()) {
12614 assert(NumDstElts == 4 &&
"Unexpected vector size");
12619 auto *HalfTy = llvm::FixedVectorType::get(
12621 Src = CGF.
Builder.CreateBitCast(Src, HalfTy);
12624 Value *Res = CGF.
Builder.CreateFPExt(Src, DstTy,
"cvtph2ps");
12626 if (Ops.size() >= 3)
12635 llvm::Type *Int32Ty = CGF.
Builder.getInt32Ty();
12636 Value *ZeroExt = CGF.
Builder.CreateZExt(Ops[0], Int32Ty);
12639 Value *BitCast = CGF.
Builder.CreateBitCast(Shl, ResultType);
12643Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
12645 llvm::Type *
Int32Ty = Builder.getInt32Ty();
12654 llvm::ArrayType::get(
Int32Ty, 1));
12664 std::tie(Index,
Value) = StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
12666 .Case(STRING, {0u,
static_cast<unsigned>(llvm::X86::ENUM)})
12668 .Case(
ALIAS, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
12670 .Case(STR, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
12672 .Case(STR, {2u,
static_cast<unsigned>(llvm::X86::ENUM)})
12673#include
"llvm/Support/X86TargetParser.def"
12675 assert(
Value != 0 &&
"Invalid CPUStr passed to CpuIs");
12678 llvm::Value *Idxs[] = {ConstantInt::get(
Int32Ty, 0),
12679 ConstantInt::get(
Int32Ty, Index)};
12680 llvm::Value *CpuValue = Builder.CreateGEP(STy, CpuModel, Idxs);
12681 CpuValue = Builder.CreateAlignedLoad(
Int32Ty, CpuValue,
12685 return Builder.CreateICmpEQ(CpuValue,
12689Value *CodeGenFunction::EmitX86CpuSupports(
const CallExpr *E) {
12692 return EmitX86CpuSupports(FeatureStr);
12696 return EmitX86CpuSupports(llvm::X86::getCpuSupportsMask(FeatureStrs));
12699llvm::Value *CodeGenFunction::EmitX86CpuSupports(
uint64_t FeaturesMask) {
12700 uint32_t Features1 = Lo_32(FeaturesMask);
12701 uint32_t Features2 = Hi_32(FeaturesMask);
12705 if (Features1 != 0) {
12713 llvm::ArrayType::get(
Int32Ty, 1));
12721 Value *Idxs[] = {Builder.getInt32(0), Builder.getInt32(3),
12722 Builder.getInt32(0)};
12723 Value *CpuFeatures = Builder.CreateGEP(STy, CpuModel, Idxs);
12724 Value *Features = Builder.CreateAlignedLoad(
Int32Ty, CpuFeatures,
12728 Value *Mask = Builder.getInt32(Features1);
12729 Value *Bitset = Builder.CreateAnd(Features, Mask);
12730 Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
12734 if (Features2 != 0) {
12736 "__cpu_features2");
12739 Value *Features = Builder.CreateAlignedLoad(
Int32Ty, CpuFeatures2,
12743 Value *Mask = Builder.getInt32(Features2);
12744 Value *Bitset = Builder.CreateAnd(Features, Mask);
12745 Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
12752Value *CodeGenFunction::EmitX86CpuInit() {
12753 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
12755 llvm::FunctionCallee Func =
12759 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
12760 return Builder.CreateCall(Func);
12765 if (BuiltinID == X86::BI__builtin_cpu_is)
12766 return EmitX86CpuIs(E);
12767 if (BuiltinID == X86::BI__builtin_cpu_supports)
12768 return EmitX86CpuSupports(E);
12769 if (BuiltinID == X86::BI__builtin_cpu_init)
12770 return EmitX86CpuInit();
12778 bool IsMaskFCmp =
false;
12779 bool IsConjFMA =
false;
12782 unsigned ICEArguments = 0;
12787 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
12789 if ((ICEArguments & (1 << i)) == 0) {
12796 Ops.push_back(llvm::ConstantInt::get(
12806 auto getCmpIntrinsicCall = [
this, &Ops](Intrinsic::ID ID,
unsigned Imm) {
12807 Ops.push_back(llvm::ConstantInt::get(
Int8Ty, Imm));
12809 return Builder.CreateCall(F, Ops);
12817 auto getVectorFCmpIR = [
this, &Ops, E](CmpInst::Predicate Pred,
12818 bool IsSignaling) {
12819 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
12822 Cmp = Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
12824 Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
12826 llvm::VectorType *IntVecTy = llvm::VectorType::getInteger(FPVecTy);
12827 Value *Sext = Builder.CreateSExt(Cmp, IntVecTy);
12828 return Builder.CreateBitCast(Sext, FPVecTy);
12831 switch (BuiltinID) {
12832 default:
return nullptr;
12833 case X86::BI_mm_prefetch: {
12836 Value *RW = ConstantInt::get(
Int32Ty, (
C->getZExtValue() >> 2) & 0x1);
12837 Value *Locality = ConstantInt::get(
Int32Ty,
C->getZExtValue() & 0x3);
12840 return Builder.CreateCall(F, {
Address, RW, Locality, Data});
12842 case X86::BI_mm_clflush: {
12843 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::x86_sse2_clflush),
12846 case X86::BI_mm_lfence: {
12847 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::x86_sse2_lfence));
12849 case X86::BI_mm_mfence: {
12850 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::x86_sse2_mfence));
12852 case X86::BI_mm_sfence: {
12853 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::x86_sse_sfence));
12855 case X86::BI_mm_pause: {
12856 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::x86_sse2_pause));
12858 case X86::BI__rdtsc: {
12861 case X86::BI__builtin_ia32_rdtscp: {
12863 Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 1),
12865 return Builder.CreateExtractValue(Call, 0);
12867 case X86::BI__builtin_ia32_lzcnt_u16:
12868 case X86::BI__builtin_ia32_lzcnt_u32:
12869 case X86::BI__builtin_ia32_lzcnt_u64: {
12871 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(
false)});
12873 case X86::BI__builtin_ia32_tzcnt_u16:
12874 case X86::BI__builtin_ia32_tzcnt_u32:
12875 case X86::BI__builtin_ia32_tzcnt_u64: {
12877 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(
false)});
12879 case X86::BI__builtin_ia32_undef128:
12880 case X86::BI__builtin_ia32_undef256:
12881 case X86::BI__builtin_ia32_undef512:
12888 case X86::BI__builtin_ia32_vec_init_v8qi:
12889 case X86::BI__builtin_ia32_vec_init_v4hi:
12890 case X86::BI__builtin_ia32_vec_init_v2si:
12893 case X86::BI__builtin_ia32_vec_ext_v2si:
12894 case X86::BI__builtin_ia32_vec_ext_v16qi:
12895 case X86::BI__builtin_ia32_vec_ext_v8hi:
12896 case X86::BI__builtin_ia32_vec_ext_v4si:
12897 case X86::BI__builtin_ia32_vec_ext_v4sf:
12898 case X86::BI__builtin_ia32_vec_ext_v2di:
12899 case X86::BI__builtin_ia32_vec_ext_v32qi:
12900 case X86::BI__builtin_ia32_vec_ext_v16hi:
12901 case X86::BI__builtin_ia32_vec_ext_v8si:
12902 case X86::BI__builtin_ia32_vec_ext_v4di: {
12906 Index &= NumElts - 1;
12909 return Builder.CreateExtractElement(Ops[0], Index);
12911 case X86::BI__builtin_ia32_vec_set_v16qi:
12912 case X86::BI__builtin_ia32_vec_set_v8hi:
12913 case X86::BI__builtin_ia32_vec_set_v4si:
12914 case X86::BI__builtin_ia32_vec_set_v2di:
12915 case X86::BI__builtin_ia32_vec_set_v32qi:
12916 case X86::BI__builtin_ia32_vec_set_v16hi:
12917 case X86::BI__builtin_ia32_vec_set_v8si:
12918 case X86::BI__builtin_ia32_vec_set_v4di: {
12922 Index &= NumElts - 1;
12925 return Builder.CreateInsertElement(Ops[0], Ops[1], Index);
12927 case X86::BI_mm_setcsr:
12928 case X86::BI__builtin_ia32_ldmxcsr: {
12930 Builder.CreateStore(Ops[0], Tmp);
12931 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::x86_sse_ldmxcsr),
12934 case X86::BI_mm_getcsr:
12935 case X86::BI__builtin_ia32_stmxcsr: {
12939 return Builder.CreateLoad(Tmp,
"stmxcsr");
12941 case X86::BI__builtin_ia32_xsave:
12942 case X86::BI__builtin_ia32_xsave64:
12943 case X86::BI__builtin_ia32_xrstor:
12944 case X86::BI__builtin_ia32_xrstor64:
12945 case X86::BI__builtin_ia32_xsaveopt:
12946 case X86::BI__builtin_ia32_xsaveopt64:
12947 case X86::BI__builtin_ia32_xrstors:
12948 case X86::BI__builtin_ia32_xrstors64:
12949 case X86::BI__builtin_ia32_xsavec:
12950 case X86::BI__builtin_ia32_xsavec64:
12951 case X86::BI__builtin_ia32_xsaves:
12952 case X86::BI__builtin_ia32_xsaves64:
12953 case X86::BI__builtin_ia32_xsetbv:
12954 case X86::BI_xsetbv: {
12956#define INTRINSIC_X86_XSAVE_ID(NAME) \
12957 case X86::BI__builtin_ia32_##NAME: \
12958 ID = Intrinsic::x86_##NAME; \
12960 switch (BuiltinID) {
12961 default: llvm_unreachable(
"Unsupported intrinsic!");
12975 case X86::BI_xsetbv:
12976 ID = Intrinsic::x86_xsetbv;
12979#undef INTRINSIC_X86_XSAVE_ID
12980 Value *Mhi = Builder.CreateTrunc(
12981 Builder.CreateLShr(Ops[1], ConstantInt::get(
Int64Ty, 32)),
Int32Ty);
12984 Ops.push_back(Mlo);
12987 case X86::BI__builtin_ia32_xgetbv:
12988 case X86::BI_xgetbv:
12989 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::x86_xgetbv), Ops);
12990 case X86::BI__builtin_ia32_storedqudi128_mask:
12991 case X86::BI__builtin_ia32_storedqusi128_mask:
12992 case X86::BI__builtin_ia32_storedquhi128_mask:
12993 case X86::BI__builtin_ia32_storedquqi128_mask:
12994 case X86::BI__builtin_ia32_storeupd128_mask:
12995 case X86::BI__builtin_ia32_storeups128_mask:
12996 case X86::BI__builtin_ia32_storedqudi256_mask:
12997 case X86::BI__builtin_ia32_storedqusi256_mask:
12998 case X86::BI__builtin_ia32_storedquhi256_mask:
12999 case X86::BI__builtin_ia32_storedquqi256_mask:
13000 case X86::BI__builtin_ia32_storeupd256_mask:
13001 case X86::BI__builtin_ia32_storeups256_mask:
13002 case X86::BI__builtin_ia32_storedqudi512_mask:
13003 case X86::BI__builtin_ia32_storedqusi512_mask:
13004 case X86::BI__builtin_ia32_storedquhi512_mask:
13005 case X86::BI__builtin_ia32_storedquqi512_mask:
13006 case X86::BI__builtin_ia32_storeupd512_mask:
13007 case X86::BI__builtin_ia32_storeups512_mask:
13010 case X86::BI__builtin_ia32_storesh128_mask:
13011 case X86::BI__builtin_ia32_storess128_mask:
13012 case X86::BI__builtin_ia32_storesd128_mask:
13015 case X86::BI__builtin_ia32_vpopcntb_128:
13016 case X86::BI__builtin_ia32_vpopcntd_128:
13017 case X86::BI__builtin_ia32_vpopcntq_128:
13018 case X86::BI__builtin_ia32_vpopcntw_128:
13019 case X86::BI__builtin_ia32_vpopcntb_256:
13020 case X86::BI__builtin_ia32_vpopcntd_256:
13021 case X86::BI__builtin_ia32_vpopcntq_256:
13022 case X86::BI__builtin_ia32_vpopcntw_256:
13023 case X86::BI__builtin_ia32_vpopcntb_512:
13024 case X86::BI__builtin_ia32_vpopcntd_512:
13025 case X86::BI__builtin_ia32_vpopcntq_512:
13026 case X86::BI__builtin_ia32_vpopcntw_512: {
13029 return Builder.CreateCall(F, Ops);
13031 case X86::BI__builtin_ia32_cvtmask2b128:
13032 case X86::BI__builtin_ia32_cvtmask2b256:
13033 case X86::BI__builtin_ia32_cvtmask2b512:
13034 case X86::BI__builtin_ia32_cvtmask2w128:
13035 case X86::BI__builtin_ia32_cvtmask2w256:
13036 case X86::BI__builtin_ia32_cvtmask2w512:
13037 case X86::BI__builtin_ia32_cvtmask2d128:
13038 case X86::BI__builtin_ia32_cvtmask2d256:
13039 case X86::BI__builtin_ia32_cvtmask2d512:
13040 case X86::BI__builtin_ia32_cvtmask2q128:
13041 case X86::BI__builtin_ia32_cvtmask2q256:
13042 case X86::BI__builtin_ia32_cvtmask2q512:
13045 case X86::BI__builtin_ia32_cvtb2mask128:
13046 case X86::BI__builtin_ia32_cvtb2mask256:
13047 case X86::BI__builtin_ia32_cvtb2mask512:
13048 case X86::BI__builtin_ia32_cvtw2mask128:
13049 case X86::BI__builtin_ia32_cvtw2mask256:
13050 case X86::BI__builtin_ia32_cvtw2mask512:
13051 case X86::BI__builtin_ia32_cvtd2mask128:
13052 case X86::BI__builtin_ia32_cvtd2mask256:
13053 case X86::BI__builtin_ia32_cvtd2mask512:
13054 case X86::BI__builtin_ia32_cvtq2mask128:
13055 case X86::BI__builtin_ia32_cvtq2mask256:
13056 case X86::BI__builtin_ia32_cvtq2mask512:
13059 case X86::BI__builtin_ia32_cvtdq2ps512_mask:
13060 case X86::BI__builtin_ia32_cvtqq2ps512_mask:
13061 case X86::BI__builtin_ia32_cvtqq2pd512_mask:
13062 case X86::BI__builtin_ia32_vcvtw2ph512_mask:
13063 case X86::BI__builtin_ia32_vcvtdq2ph512_mask:
13064 case X86::BI__builtin_ia32_vcvtqq2ph512_mask:
13066 case X86::BI__builtin_ia32_cvtudq2ps512_mask:
13067 case X86::BI__builtin_ia32_cvtuqq2ps512_mask:
13068 case X86::BI__builtin_ia32_cvtuqq2pd512_mask:
13069 case X86::BI__builtin_ia32_vcvtuw2ph512_mask:
13070 case X86::BI__builtin_ia32_vcvtudq2ph512_mask:
13071 case X86::BI__builtin_ia32_vcvtuqq2ph512_mask:
13074 case X86::BI__builtin_ia32_vfmaddss3:
13075 case X86::BI__builtin_ia32_vfmaddsd3:
13076 case X86::BI__builtin_ia32_vfmaddsh3_mask:
13077 case X86::BI__builtin_ia32_vfmaddss3_mask:
13078 case X86::BI__builtin_ia32_vfmaddsd3_mask:
13080 case X86::BI__builtin_ia32_vfmaddss:
13081 case X86::BI__builtin_ia32_vfmaddsd:
13083 Constant::getNullValue(Ops[0]->
getType()));
13084 case X86::BI__builtin_ia32_vfmaddsh3_maskz:
13085 case X86::BI__builtin_ia32_vfmaddss3_maskz:
13086 case X86::BI__builtin_ia32_vfmaddsd3_maskz:
13088 case X86::BI__builtin_ia32_vfmaddsh3_mask3:
13089 case X86::BI__builtin_ia32_vfmaddss3_mask3:
13090 case X86::BI__builtin_ia32_vfmaddsd3_mask3:
13092 case X86::BI__builtin_ia32_vfmsubsh3_mask3:
13093 case X86::BI__builtin_ia32_vfmsubss3_mask3:
13094 case X86::BI__builtin_ia32_vfmsubsd3_mask3:
13097 case X86::BI__builtin_ia32_vfmaddph:
13098 case X86::BI__builtin_ia32_vfmaddps:
13099 case X86::BI__builtin_ia32_vfmaddpd:
13100 case X86::BI__builtin_ia32_vfmaddph256:
13101 case X86::BI__builtin_ia32_vfmaddps256:
13102 case X86::BI__builtin_ia32_vfmaddpd256:
13103 case X86::BI__builtin_ia32_vfmaddph512_mask:
13104 case X86::BI__builtin_ia32_vfmaddph512_maskz:
13105 case X86::BI__builtin_ia32_vfmaddph512_mask3:
13106 case X86::BI__builtin_ia32_vfmaddps512_mask:
13107 case X86::BI__builtin_ia32_vfmaddps512_maskz:
13108 case X86::BI__builtin_ia32_vfmaddps512_mask3:
13109 case X86::BI__builtin_ia32_vfmsubps512_mask3:
13110 case X86::BI__builtin_ia32_vfmaddpd512_mask:
13111 case X86::BI__builtin_ia32_vfmaddpd512_maskz:
13112 case X86::BI__builtin_ia32_vfmaddpd512_mask3:
13113 case X86::BI__builtin_ia32_vfmsubpd512_mask3:
13114 case X86::BI__builtin_ia32_vfmsubph512_mask3:
13116 case X86::BI__builtin_ia32_vfmaddsubph512_mask:
13117 case X86::BI__builtin_ia32_vfmaddsubph512_maskz:
13118 case X86::BI__builtin_ia32_vfmaddsubph512_mask3:
13119 case X86::BI__builtin_ia32_vfmsubaddph512_mask3:
13120 case X86::BI__builtin_ia32_vfmaddsubps512_mask:
13121 case X86::BI__builtin_ia32_vfmaddsubps512_maskz:
13122 case X86::BI__builtin_ia32_vfmaddsubps512_mask3:
13123 case X86::BI__builtin_ia32_vfmsubaddps512_mask3:
13124 case X86::BI__builtin_ia32_vfmaddsubpd512_mask:
13125 case X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
13126 case X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
13127 case X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
13130 case X86::BI__builtin_ia32_movdqa32store128_mask:
13131 case X86::BI__builtin_ia32_movdqa64store128_mask:
13132 case X86::BI__builtin_ia32_storeaps128_mask:
13133 case X86::BI__builtin_ia32_storeapd128_mask:
13134 case X86::BI__builtin_ia32_movdqa32store256_mask:
13135 case X86::BI__builtin_ia32_movdqa64store256_mask:
13136 case X86::BI__builtin_ia32_storeaps256_mask:
13137 case X86::BI__builtin_ia32_storeapd256_mask:
13138 case X86::BI__builtin_ia32_movdqa32store512_mask:
13139 case X86::BI__builtin_ia32_movdqa64store512_mask:
13140 case X86::BI__builtin_ia32_storeaps512_mask:
13141 case X86::BI__builtin_ia32_storeapd512_mask:
13146 case X86::BI__builtin_ia32_loadups128_mask:
13147 case X86::BI__builtin_ia32_loadups256_mask:
13148 case X86::BI__builtin_ia32_loadups512_mask:
13149 case X86::BI__builtin_ia32_loadupd128_mask:
13150 case X86::BI__builtin_ia32_loadupd256_mask:
13151 case X86::BI__builtin_ia32_loadupd512_mask:
13152 case X86::BI__builtin_ia32_loaddquqi128_mask:
13153 case X86::BI__builtin_ia32_loaddquqi256_mask:
13154 case X86::BI__builtin_ia32_loaddquqi512_mask:
13155 case X86::BI__builtin_ia32_loaddquhi128_mask:
13156 case X86::BI__builtin_ia32_loaddquhi256_mask:
13157 case X86::BI__builtin_ia32_loaddquhi512_mask:
13158 case X86::BI__builtin_ia32_loaddqusi128_mask:
13159 case X86::BI__builtin_ia32_loaddqusi256_mask:
13160 case X86::BI__builtin_ia32_loaddqusi512_mask:
13161 case X86::BI__builtin_ia32_loaddqudi128_mask:
13162 case X86::BI__builtin_ia32_loaddqudi256_mask:
13163 case X86::BI__builtin_ia32_loaddqudi512_mask:
13166 case X86::BI__builtin_ia32_loadsh128_mask:
13167 case X86::BI__builtin_ia32_loadss128_mask:
13168 case X86::BI__builtin_ia32_loadsd128_mask:
13171 case X86::BI__builtin_ia32_loadaps128_mask:
13172 case X86::BI__builtin_ia32_loadaps256_mask:
13173 case X86::BI__builtin_ia32_loadaps512_mask:
13174 case X86::BI__builtin_ia32_loadapd128_mask:
13175 case X86::BI__builtin_ia32_loadapd256_mask:
13176 case X86::BI__builtin_ia32_loadapd512_mask:
13177 case X86::BI__builtin_ia32_movdqa32load128_mask:
13178 case X86::BI__builtin_ia32_movdqa32load256_mask:
13179 case X86::BI__builtin_ia32_movdqa32load512_mask:
13180 case X86::BI__builtin_ia32_movdqa64load128_mask:
13181 case X86::BI__builtin_ia32_movdqa64load256_mask:
13182 case X86::BI__builtin_ia32_movdqa64load512_mask:
13187 case X86::BI__builtin_ia32_expandloaddf128_mask:
13188 case X86::BI__builtin_ia32_expandloaddf256_mask:
13189 case X86::BI__builtin_ia32_expandloaddf512_mask:
13190 case X86::BI__builtin_ia32_expandloadsf128_mask:
13191 case X86::BI__builtin_ia32_expandloadsf256_mask:
13192 case X86::BI__builtin_ia32_expandloadsf512_mask:
13193 case X86::BI__builtin_ia32_expandloaddi128_mask:
13194 case X86::BI__builtin_ia32_expandloaddi256_mask:
13195 case X86::BI__builtin_ia32_expandloaddi512_mask:
13196 case X86::BI__builtin_ia32_expandloadsi128_mask:
13197 case X86::BI__builtin_ia32_expandloadsi256_mask:
13198 case X86::BI__builtin_ia32_expandloadsi512_mask:
13199 case X86::BI__builtin_ia32_expandloadhi128_mask:
13200 case X86::BI__builtin_ia32_expandloadhi256_mask:
13201 case X86::BI__builtin_ia32_expandloadhi512_mask:
13202 case X86::BI__builtin_ia32_expandloadqi128_mask:
13203 case X86::BI__builtin_ia32_expandloadqi256_mask:
13204 case X86::BI__builtin_ia32_expandloadqi512_mask:
13207 case X86::BI__builtin_ia32_compressstoredf128_mask:
13208 case X86::BI__builtin_ia32_compressstoredf256_mask:
13209 case X86::BI__builtin_ia32_compressstoredf512_mask:
13210 case X86::BI__builtin_ia32_compressstoresf128_mask:
13211 case X86::BI__builtin_ia32_compressstoresf256_mask:
13212 case X86::BI__builtin_ia32_compressstoresf512_mask:
13213 case X86::BI__builtin_ia32_compressstoredi128_mask:
13214 case X86::BI__builtin_ia32_compressstoredi256_mask:
13215 case X86::BI__builtin_ia32_compressstoredi512_mask:
13216 case X86::BI__builtin_ia32_compressstoresi128_mask:
13217 case X86::BI__builtin_ia32_compressstoresi256_mask:
13218 case X86::BI__builtin_ia32_compressstoresi512_mask:
13219 case X86::BI__builtin_ia32_compressstorehi128_mask:
13220 case X86::BI__builtin_ia32_compressstorehi256_mask:
13221 case X86::BI__builtin_ia32_compressstorehi512_mask:
13222 case X86::BI__builtin_ia32_compressstoreqi128_mask:
13223 case X86::BI__builtin_ia32_compressstoreqi256_mask:
13224 case X86::BI__builtin_ia32_compressstoreqi512_mask:
13227 case X86::BI__builtin_ia32_expanddf128_mask:
13228 case X86::BI__builtin_ia32_expanddf256_mask:
13229 case X86::BI__builtin_ia32_expanddf512_mask:
13230 case X86::BI__builtin_ia32_expandsf128_mask:
13231 case X86::BI__builtin_ia32_expandsf256_mask:
13232 case X86::BI__builtin_ia32_expandsf512_mask:
13233 case X86::BI__builtin_ia32_expanddi128_mask:
13234 case X86::BI__builtin_ia32_expanddi256_mask:
13235 case X86::BI__builtin_ia32_expanddi512_mask:
13236 case X86::BI__builtin_ia32_expandsi128_mask:
13237 case X86::BI__builtin_ia32_expandsi256_mask:
13238 case X86::BI__builtin_ia32_expandsi512_mask:
13239 case X86::BI__builtin_ia32_expandhi128_mask:
13240 case X86::BI__builtin_ia32_expandhi256_mask:
13241 case X86::BI__builtin_ia32_expandhi512_mask:
13242 case X86::BI__builtin_ia32_expandqi128_mask:
13243 case X86::BI__builtin_ia32_expandqi256_mask:
13244 case X86::BI__builtin_ia32_expandqi512_mask:
13247 case X86::BI__builtin_ia32_compressdf128_mask:
13248 case X86::BI__builtin_ia32_compressdf256_mask:
13249 case X86::BI__builtin_ia32_compressdf512_mask:
13250 case X86::BI__builtin_ia32_compresssf128_mask:
13251 case X86::BI__builtin_ia32_compresssf256_mask:
13252 case X86::BI__builtin_ia32_compresssf512_mask:
13253 case X86::BI__builtin_ia32_compressdi128_mask:
13254 case X86::BI__builtin_ia32_compressdi256_mask:
13255 case X86::BI__builtin_ia32_compressdi512_mask:
13256 case X86::BI__builtin_ia32_compresssi128_mask:
13257 case X86::BI__builtin_ia32_compresssi256_mask:
13258 case X86::BI__builtin_ia32_compresssi512_mask:
13259 case X86::BI__builtin_ia32_compresshi128_mask:
13260 case X86::BI__builtin_ia32_compresshi256_mask:
13261 case X86::BI__builtin_ia32_compresshi512_mask:
13262 case X86::BI__builtin_ia32_compressqi128_mask:
13263 case X86::BI__builtin_ia32_compressqi256_mask:
13264 case X86::BI__builtin_ia32_compressqi512_mask:
13267 case X86::BI__builtin_ia32_gather3div2df:
13268 case X86::BI__builtin_ia32_gather3div2di:
13269 case X86::BI__builtin_ia32_gather3div4df:
13270 case X86::BI__builtin_ia32_gather3div4di:
13271 case X86::BI__builtin_ia32_gather3div4sf:
13272 case X86::BI__builtin_ia32_gather3div4si:
13273 case X86::BI__builtin_ia32_gather3div8sf:
13274 case X86::BI__builtin_ia32_gather3div8si:
13275 case X86::BI__builtin_ia32_gather3siv2df:
13276 case X86::BI__builtin_ia32_gather3siv2di:
13277 case X86::BI__builtin_ia32_gather3siv4df:
13278 case X86::BI__builtin_ia32_gather3siv4di:
13279 case X86::BI__builtin_ia32_gather3siv4sf:
13280 case X86::BI__builtin_ia32_gather3siv4si:
13281 case X86::BI__builtin_ia32_gather3siv8sf:
13282 case X86::BI__builtin_ia32_gather3siv8si:
13283 case X86::BI__builtin_ia32_gathersiv8df:
13284 case X86::BI__builtin_ia32_gathersiv16sf:
13285 case X86::BI__builtin_ia32_gatherdiv8df:
13286 case X86::BI__builtin_ia32_gatherdiv16sf:
13287 case X86::BI__builtin_ia32_gathersiv8di:
13288 case X86::BI__builtin_ia32_gathersiv16si:
13289 case X86::BI__builtin_ia32_gatherdiv8di:
13290 case X86::BI__builtin_ia32_gatherdiv16si: {
13292 switch (BuiltinID) {
13293 default: llvm_unreachable(
"Unexpected builtin");
13294 case X86::BI__builtin_ia32_gather3div2df:
13295 IID = Intrinsic::x86_avx512_mask_gather3div2_df;
13297 case X86::BI__builtin_ia32_gather3div2di:
13298 IID = Intrinsic::x86_avx512_mask_gather3div2_di;
13300 case X86::BI__builtin_ia32_gather3div4df:
13301 IID = Intrinsic::x86_avx512_mask_gather3div4_df;
13303 case X86::BI__builtin_ia32_gather3div4di:
13304 IID = Intrinsic::x86_avx512_mask_gather3div4_di;
13306 case X86::BI__builtin_ia32_gather3div4sf:
13307 IID = Intrinsic::x86_avx512_mask_gather3div4_sf;
13309 case X86::BI__builtin_ia32_gather3div4si:
13310 IID = Intrinsic::x86_avx512_mask_gather3div4_si;
13312 case X86::BI__builtin_ia32_gather3div8sf:
13313 IID = Intrinsic::x86_avx512_mask_gather3div8_sf;
13315 case X86::BI__builtin_ia32_gather3div8si:
13316 IID = Intrinsic::x86_avx512_mask_gather3div8_si;
13318 case X86::BI__builtin_ia32_gather3siv2df:
13319 IID = Intrinsic::x86_avx512_mask_gather3siv2_df;
13321 case X86::BI__builtin_ia32_gather3siv2di:
13322 IID = Intrinsic::x86_avx512_mask_gather3siv2_di;
13324 case X86::BI__builtin_ia32_gather3siv4df:
13325 IID = Intrinsic::x86_avx512_mask_gather3siv4_df;
13327 case X86::BI__builtin_ia32_gather3siv4di:
13328 IID = Intrinsic::x86_avx512_mask_gather3siv4_di;
13330 case X86::BI__builtin_ia32_gather3siv4sf:
13331 IID = Intrinsic::x86_avx512_mask_gather3siv4_sf;
13333 case X86::BI__builtin_ia32_gather3siv4si:
13334 IID = Intrinsic::x86_avx512_mask_gather3siv4_si;
13336 case X86::BI__builtin_ia32_gather3siv8sf:
13337 IID = Intrinsic::x86_avx512_mask_gather3siv8_sf;
13339 case X86::BI__builtin_ia32_gather3siv8si:
13340 IID = Intrinsic::x86_avx512_mask_gather3siv8_si;
13342 case X86::BI__builtin_ia32_gathersiv8df:
13343 IID = Intrinsic::x86_avx512_mask_gather_dpd_512;
13345 case X86::BI__builtin_ia32_gathersiv16sf:
13346 IID = Intrinsic::x86_avx512_mask_gather_dps_512;
13348 case X86::BI__builtin_ia32_gatherdiv8df:
13349 IID = Intrinsic::x86_avx512_mask_gather_qpd_512;
13351 case X86::BI__builtin_ia32_gatherdiv16sf:
13352 IID = Intrinsic::x86_avx512_mask_gather_qps_512;
13354 case X86::BI__builtin_ia32_gathersiv8di:
13355 IID = Intrinsic::x86_avx512_mask_gather_dpq_512;
13357 case X86::BI__builtin_ia32_gathersiv16si:
13358 IID = Intrinsic::x86_avx512_mask_gather_dpi_512;
13360 case X86::BI__builtin_ia32_gatherdiv8di:
13361 IID = Intrinsic::x86_avx512_mask_gather_qpq_512;
13363 case X86::BI__builtin_ia32_gatherdiv16si:
13364 IID = Intrinsic::x86_avx512_mask_gather_qpi_512;
13368 unsigned MinElts = std::min(
13369 cast<llvm::FixedVectorType>(Ops[0]->
getType())->getNumElements(),
13370 cast<llvm::FixedVectorType>(Ops[2]->
getType())->getNumElements());
13373 return Builder.CreateCall(Intr, Ops);
13376 case X86::BI__builtin_ia32_scattersiv8df:
13377 case X86::BI__builtin_ia32_scattersiv16sf:
13378 case X86::BI__builtin_ia32_scatterdiv8df:
13379 case X86::BI__builtin_ia32_scatterdiv16sf:
13380 case X86::BI__builtin_ia32_scattersiv8di:
13381 case X86::BI__builtin_ia32_scattersiv16si:
13382 case X86::BI__builtin_ia32_scatterdiv8di:
13383 case X86::BI__builtin_ia32_scatterdiv16si:
13384 case X86::BI__builtin_ia32_scatterdiv2df:
13385 case X86::BI__builtin_ia32_scatterdiv2di:
13386 case X86::BI__builtin_ia32_scatterdiv4df:
13387 case X86::BI__builtin_ia32_scatterdiv4di:
13388 case X86::BI__builtin_ia32_scatterdiv4sf:
13389 case X86::BI__builtin_ia32_scatterdiv4si:
13390 case X86::BI__builtin_ia32_scatterdiv8sf:
13391 case X86::BI__builtin_ia32_scatterdiv8si:
13392 case X86::BI__builtin_ia32_scattersiv2df:
13393 case X86::BI__builtin_ia32_scattersiv2di:
13394 case X86::BI__builtin_ia32_scattersiv4df:
13395 case X86::BI__builtin_ia32_scattersiv4di:
13396 case X86::BI__builtin_ia32_scattersiv4sf:
13397 case X86::BI__builtin_ia32_scattersiv4si:
13398 case X86::BI__builtin_ia32_scattersiv8sf:
13399 case X86::BI__builtin_ia32_scattersiv8si: {
13401 switch (BuiltinID) {
13402 default: llvm_unreachable(
"Unexpected builtin");
13403 case X86::BI__builtin_ia32_scattersiv8df:
13404 IID = Intrinsic::x86_avx512_mask_scatter_dpd_512;
13406 case X86::BI__builtin_ia32_scattersiv16sf:
13407 IID = Intrinsic::x86_avx512_mask_scatter_dps_512;
13409 case X86::BI__builtin_ia32_scatterdiv8df:
13410 IID = Intrinsic::x86_avx512_mask_scatter_qpd_512;
13412 case X86::BI__builtin_ia32_scatterdiv16sf:
13413 IID = Intrinsic::x86_avx512_mask_scatter_qps_512;
13415 case X86::BI__builtin_ia32_scattersiv8di:
13416 IID = Intrinsic::x86_avx512_mask_scatter_dpq_512;
13418 case X86::BI__builtin_ia32_scattersiv16si:
13419 IID = Intrinsic::x86_avx512_mask_scatter_dpi_512;
13421 case X86::BI__builtin_ia32_scatterdiv8di:
13422 IID = Intrinsic::x86_avx512_mask_scatter_qpq_512;
13424 case X86::BI__builtin_ia32_scatterdiv16si:
13425 IID = Intrinsic::x86_avx512_mask_scatter_qpi_512;
13427 case X86::BI__builtin_ia32_scatterdiv2df:
13428 IID = Intrinsic::x86_avx512_mask_scatterdiv2_df;
13430 case X86::BI__builtin_ia32_scatterdiv2di:
13431 IID = Intrinsic::x86_avx512_mask_scatterdiv2_di;
13433 case X86::BI__builtin_ia32_scatterdiv4df:
13434 IID = Intrinsic::x86_avx512_mask_scatterdiv4_df;
13436 case X86::BI__builtin_ia32_scatterdiv4di:
13437 IID = Intrinsic::x86_avx512_mask_scatterdiv4_di;
13439 case X86::BI__builtin_ia32_scatterdiv4sf:
13440 IID = Intrinsic::x86_avx512_mask_scatterdiv4_sf;
13442 case X86::BI__builtin_ia32_scatterdiv4si:
13443 IID = Intrinsic::x86_avx512_mask_scatterdiv4_si;
13445 case X86::BI__builtin_ia32_scatterdiv8sf:
13446 IID = Intrinsic::x86_avx512_mask_scatterdiv8_sf;
13448 case X86::BI__builtin_ia32_scatterdiv8si:
13449 IID = Intrinsic::x86_avx512_mask_scatterdiv8_si;
13451 case X86::BI__builtin_ia32_scattersiv2df:
13452 IID = Intrinsic::x86_avx512_mask_scattersiv2_df;
13454 case X86::BI__builtin_ia32_scattersiv2di:
13455 IID = Intrinsic::x86_avx512_mask_scattersiv2_di;
13457 case X86::BI__builtin_ia32_scattersiv4df:
13458 IID = Intrinsic::x86_avx512_mask_scattersiv4_df;
13460 case X86::BI__builtin_ia32_scattersiv4di:
13461 IID = Intrinsic::x86_avx512_mask_scattersiv4_di;
13463 case X86::BI__builtin_ia32_scattersiv4sf:
13464 IID = Intrinsic::x86_avx512_mask_scattersiv4_sf;
13466 case X86::BI__builtin_ia32_scattersiv4si:
13467 IID = Intrinsic::x86_avx512_mask_scattersiv4_si;
13469 case X86::BI__builtin_ia32_scattersiv8sf:
13470 IID = Intrinsic::x86_avx512_mask_scattersiv8_sf;
13472 case X86::BI__builtin_ia32_scattersiv8si:
13473 IID = Intrinsic::x86_avx512_mask_scattersiv8_si;
13477 unsigned MinElts = std::min(
13478 cast<llvm::FixedVectorType>(Ops[2]->
getType())->getNumElements(),
13479 cast<llvm::FixedVectorType>(Ops[3]->
getType())->getNumElements());
13482 return Builder.CreateCall(Intr, Ops);
13485 case X86::BI__builtin_ia32_vextractf128_pd256:
13486 case X86::BI__builtin_ia32_vextractf128_ps256:
13487 case X86::BI__builtin_ia32_vextractf128_si256:
13488 case X86::BI__builtin_ia32_extract128i256:
13489 case X86::BI__builtin_ia32_extractf64x4_mask:
13490 case X86::BI__builtin_ia32_extractf32x4_mask:
13491 case X86::BI__builtin_ia32_extracti64x4_mask:
13492 case X86::BI__builtin_ia32_extracti32x4_mask:
13493 case X86::BI__builtin_ia32_extractf32x8_mask:
13494 case X86::BI__builtin_ia32_extracti32x8_mask:
13495 case X86::BI__builtin_ia32_extractf32x4_256_mask:
13496 case X86::BI__builtin_ia32_extracti32x4_256_mask:
13497 case X86::BI__builtin_ia32_extractf64x2_256_mask:
13498 case X86::BI__builtin_ia32_extracti64x2_256_mask:
13499 case X86::BI__builtin_ia32_extractf64x2_512_mask:
13500 case X86::BI__builtin_ia32_extracti64x2_512_mask: {
13502 unsigned NumElts = DstTy->getNumElements();
13503 unsigned SrcNumElts =
13505 unsigned SubVectors = SrcNumElts / NumElts;
13507 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
13508 Index &= SubVectors - 1;
13512 for (
unsigned i = 0; i != NumElts; ++i)
13513 Indices[i] = i + Index;
13515 Value *Res = Builder.CreateShuffleVector(Ops[0],
13516 makeArrayRef(Indices, NumElts),
13519 if (Ops.size() == 4)
13524 case X86::BI__builtin_ia32_vinsertf128_pd256:
13525 case X86::BI__builtin_ia32_vinsertf128_ps256:
13526 case X86::BI__builtin_ia32_vinsertf128_si256:
13527 case X86::BI__builtin_ia32_insert128i256:
13528 case X86::BI__builtin_ia32_insertf64x4:
13529 case X86::BI__builtin_ia32_insertf32x4:
13530 case X86::BI__builtin_ia32_inserti64x4:
13531 case X86::BI__builtin_ia32_inserti32x4:
13532 case X86::BI__builtin_ia32_insertf32x8:
13533 case X86::BI__builtin_ia32_inserti32x8:
13534 case X86::BI__builtin_ia32_insertf32x4_256:
13535 case X86::BI__builtin_ia32_inserti32x4_256:
13536 case X86::BI__builtin_ia32_insertf64x2_256:
13537 case X86::BI__builtin_ia32_inserti64x2_256:
13538 case X86::BI__builtin_ia32_insertf64x2_512:
13539 case X86::BI__builtin_ia32_inserti64x2_512: {
13540 unsigned DstNumElts =
13542 unsigned SrcNumElts =
13544 unsigned SubVectors = DstNumElts / SrcNumElts;
13546 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
13547 Index &= SubVectors - 1;
13548 Index *= SrcNumElts;
13551 for (
unsigned i = 0; i != DstNumElts; ++i)
13552 Indices[i] = (i >= SrcNumElts) ? SrcNumElts + (i % SrcNumElts) : i;
13554 Value *Op1 = Builder.CreateShuffleVector(Ops[1],
13555 makeArrayRef(Indices, DstNumElts),
13558 for (
unsigned i = 0; i != DstNumElts; ++i) {
13559 if (i >= Index && i < (Index + SrcNumElts))
13560 Indices[i] = (i - Index) + DstNumElts;
13565 return Builder.CreateShuffleVector(Ops[0], Op1,
13566 makeArrayRef(Indices, DstNumElts),
13569 case X86::BI__builtin_ia32_pmovqd512_mask:
13570 case X86::BI__builtin_ia32_pmovwb512_mask: {
13571 Value *Res = Builder.CreateTrunc(Ops[0], Ops[1]->
getType());
13574 case X86::BI__builtin_ia32_pmovdb512_mask:
13575 case X86::BI__builtin_ia32_pmovdw512_mask:
13576 case X86::BI__builtin_ia32_pmovqw512_mask: {
13577 if (
const auto *
C = dyn_cast<Constant>(Ops[2]))
13578 if (
C->isAllOnesValue())
13579 return Builder.CreateTrunc(Ops[0], Ops[1]->
getType());
13582 switch (BuiltinID) {
13583 default: llvm_unreachable(
"Unsupported intrinsic!");
13584 case X86::BI__builtin_ia32_pmovdb512_mask:
13585 IID = Intrinsic::x86_avx512_mask_pmov_db_512;
13587 case X86::BI__builtin_ia32_pmovdw512_mask:
13588 IID = Intrinsic::x86_avx512_mask_pmov_dw_512;
13590 case X86::BI__builtin_ia32_pmovqw512_mask:
13591 IID = Intrinsic::x86_avx512_mask_pmov_qw_512;
13596 return Builder.CreateCall(Intr, Ops);
13598 case X86::BI__builtin_ia32_pblendw128:
13599 case X86::BI__builtin_ia32_blendpd:
13600 case X86::BI__builtin_ia32_blendps:
13601 case X86::BI__builtin_ia32_blendpd256:
13602 case X86::BI__builtin_ia32_blendps256:
13603 case X86::BI__builtin_ia32_pblendw256:
13604 case X86::BI__builtin_ia32_pblendd128:
13605 case X86::BI__builtin_ia32_pblendd256: {
13613 for (
unsigned i = 0; i != NumElts; ++i)
13614 Indices[i] = ((Imm >> (i % 8)) & 0x1) ? NumElts + i : i;
13616 return Builder.CreateShuffleVector(Ops[0], Ops[1],
13617 makeArrayRef(Indices, NumElts),
13620 case X86::BI__builtin_ia32_pshuflw:
13621 case X86::BI__builtin_ia32_pshuflw256:
13622 case X86::BI__builtin_ia32_pshuflw512: {
13625 unsigned NumElts = Ty->getNumElements();
13628 Imm = (Imm & 0xff) * 0x01010101;
13631 for (
unsigned l = 0; l != NumElts; l += 8) {
13632 for (
unsigned i = 0; i != 4; ++i) {
13633 Indices[l + i] = l + (Imm & 3);
13636 for (
unsigned i = 4; i != 8; ++i)
13637 Indices[l + i] = l + i;
13640 return Builder.CreateShuffleVector(Ops[0], makeArrayRef(Indices, NumElts),
13643 case X86::BI__builtin_ia32_pshufhw:
13644 case X86::BI__builtin_ia32_pshufhw256:
13645 case X86::BI__builtin_ia32_pshufhw512: {
13648 unsigned NumElts = Ty->getNumElements();
13651 Imm = (Imm & 0xff) * 0x01010101;
13654 for (
unsigned l = 0; l != NumElts; l += 8) {
13655 for (
unsigned i = 0; i != 4; ++i)
13656 Indices[l + i] = l + i;
13657 for (
unsigned i = 4; i != 8; ++i) {
13658 Indices[l + i] = l + 4 + (Imm & 3);
13663 return Builder.CreateShuffleVector(Ops[0], makeArrayRef(Indices, NumElts),
13666 case X86::BI__builtin_ia32_pshufd:
13667 case X86::BI__builtin_ia32_pshufd256:
13668 case X86::BI__builtin_ia32_pshufd512:
13669 case X86::BI__builtin_ia32_vpermilpd:
13670 case X86::BI__builtin_ia32_vpermilps:
13671 case X86::BI__builtin_ia32_vpermilpd256:
13672 case X86::BI__builtin_ia32_vpermilps256:
13673 case X86::BI__builtin_ia32_vpermilpd512:
13674 case X86::BI__builtin_ia32_vpermilps512: {
13677 unsigned NumElts = Ty->getNumElements();
13678 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
13679 unsigned NumLaneElts = NumElts / NumLanes;
13682 Imm = (Imm & 0xff) * 0x01010101;
13685 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
13686 for (
unsigned i = 0; i != NumLaneElts; ++i) {
13687 Indices[i + l] = (Imm % NumLaneElts) + l;
13688 Imm /= NumLaneElts;
13692 return Builder.CreateShuffleVector(Ops[0], makeArrayRef(Indices, NumElts),
13695 case X86::BI__builtin_ia32_shufpd:
13696 case X86::BI__builtin_ia32_shufpd256:
13697 case X86::BI__builtin_ia32_shufpd512:
13698 case X86::BI__builtin_ia32_shufps:
13699 case X86::BI__builtin_ia32_shufps256:
13700 case X86::BI__builtin_ia32_shufps512: {
13703 unsigned NumElts = Ty->getNumElements();
13704 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
13705 unsigned NumLaneElts = NumElts / NumLanes;
13708 Imm = (Imm & 0xff) * 0x01010101;
13711 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
13712 for (
unsigned i = 0; i != NumLaneElts; ++i) {
13713 unsigned Index = Imm % NumLaneElts;
13714 Imm /= NumLaneElts;
13715 if (i >= (NumLaneElts / 2))
13717 Indices[l + i] = l + Index;
13721 return Builder.CreateShuffleVector(Ops[0], Ops[1],
13722 makeArrayRef(Indices, NumElts),
13725 case X86::BI__builtin_ia32_permdi256:
13726 case X86::BI__builtin_ia32_permdf256:
13727 case X86::BI__builtin_ia32_permdi512:
13728 case X86::BI__builtin_ia32_permdf512: {
13731 unsigned NumElts = Ty->getNumElements();
13735 for (
unsigned l = 0; l != NumElts; l += 4)
13736 for (
unsigned i = 0; i != 4; ++i)
13737 Indices[l + i] = l + ((Imm >> (2 * i)) & 0x3);
13739 return Builder.CreateShuffleVector(Ops[0], makeArrayRef(Indices, NumElts),
13742 case X86::BI__builtin_ia32_palignr128:
13743 case X86::BI__builtin_ia32_palignr256:
13744 case X86::BI__builtin_ia32_palignr512: {
13749 assert(NumElts % 16 == 0);
13753 if (ShiftVal >= 32)
13758 if (ShiftVal > 16) {
13761 Ops[0] = llvm::Constant::getNullValue(Ops[0]->
getType());
13766 for (
unsigned l = 0; l != NumElts; l += 16) {
13767 for (
unsigned i = 0; i != 16; ++i) {
13768 unsigned Idx = ShiftVal + i;
13770 Idx += NumElts - 16;
13771 Indices[l + i] = Idx + l;
13775 return Builder.CreateShuffleVector(Ops[1], Ops[0],
13776 makeArrayRef(Indices, NumElts),
13779 case X86::BI__builtin_ia32_alignd128:
13780 case X86::BI__builtin_ia32_alignd256:
13781 case X86::BI__builtin_ia32_alignd512:
13782 case X86::BI__builtin_ia32_alignq128:
13783 case X86::BI__builtin_ia32_alignq256:
13784 case X86::BI__builtin_ia32_alignq512: {
13790 ShiftVal &= NumElts - 1;
13793 for (
unsigned i = 0; i != NumElts; ++i)
13794 Indices[i] = i + ShiftVal;
13796 return Builder.CreateShuffleVector(Ops[1], Ops[0],
13797 makeArrayRef(Indices, NumElts),
13800 case X86::BI__builtin_ia32_shuf_f32x4_256:
13801 case X86::BI__builtin_ia32_shuf_f64x2_256:
13802 case X86::BI__builtin_ia32_shuf_i32x4_256:
13803 case X86::BI__builtin_ia32_shuf_i64x2_256:
13804 case X86::BI__builtin_ia32_shuf_f32x4:
13805 case X86::BI__builtin_ia32_shuf_f64x2:
13806 case X86::BI__builtin_ia32_shuf_i32x4:
13807 case X86::BI__builtin_ia32_shuf_i64x2: {
13810 unsigned NumElts = Ty->getNumElements();
13811 unsigned NumLanes = Ty->getPrimitiveSizeInBits() == 512 ? 4 : 2;
13812 unsigned NumLaneElts = NumElts / NumLanes;
13815 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
13816 unsigned Index = (Imm % NumLanes) * NumLaneElts;
13818 if (l >= (NumElts / 2))
13820 for (
unsigned i = 0; i != NumLaneElts; ++i) {
13821 Indices[l + i] = Index + i;
13825 return Builder.CreateShuffleVector(Ops[0], Ops[1],
13826 makeArrayRef(Indices, NumElts),
13830 case X86::BI__builtin_ia32_vperm2f128_pd256:
13831 case X86::BI__builtin_ia32_vperm2f128_ps256:
13832 case X86::BI__builtin_ia32_vperm2f128_si256:
13833 case X86::BI__builtin_ia32_permti256: {
13845 for (
unsigned l = 0; l != 2; ++l) {
13847 if (Imm & (1 << ((l * 4) + 3)))
13848 OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->
getType());
13849 else if (Imm & (1 << ((l * 4) + 1)))
13850 OutOps[l] = Ops[1];
13852 OutOps[l] = Ops[0];
13854 for (
unsigned i = 0; i != NumElts/2; ++i) {
13856 unsigned Idx = (l * NumElts) + i;
13859 if (Imm & (1 << (l * 4)))
13861 Indices[(l * (NumElts/2)) + i] = Idx;
13865 return Builder.CreateShuffleVector(OutOps[0], OutOps[1],
13866 makeArrayRef(Indices, NumElts),
13870 case X86::BI__builtin_ia32_pslldqi128_byteshift:
13871 case X86::BI__builtin_ia32_pslldqi256_byteshift:
13872 case X86::BI__builtin_ia32_pslldqi512_byteshift: {
13876 unsigned NumElts = ResultType->getNumElements() * 8;
13879 if (ShiftVal >= 16)
13880 return llvm::Constant::getNullValue(ResultType);
13884 for (
unsigned l = 0; l != NumElts; l += 16) {
13885 for (
unsigned i = 0; i != 16; ++i) {
13886 unsigned Idx = NumElts + i - ShiftVal;
13887 if (Idx < NumElts) Idx -= NumElts - 16;
13888 Indices[l + i] = Idx + l;
13892 auto *VecTy = llvm::FixedVectorType::get(
Int8Ty, NumElts);
13893 Value *
Cast = Builder.CreateBitCast(Ops[0], VecTy,
"cast");
13894 Value *
Zero = llvm::Constant::getNullValue(VecTy);
13895 Value *SV = Builder.CreateShuffleVector(Zero, Cast,
13896 makeArrayRef(Indices, NumElts),
13898 return Builder.CreateBitCast(SV, Ops[0]->
getType(),
"cast");
13900 case X86::BI__builtin_ia32_psrldqi128_byteshift:
13901 case X86::BI__builtin_ia32_psrldqi256_byteshift:
13902 case X86::BI__builtin_ia32_psrldqi512_byteshift: {
13906 unsigned NumElts = ResultType->getNumElements() * 8;
13909 if (ShiftVal >= 16)
13910 return llvm::Constant::getNullValue(ResultType);
13914 for (
unsigned l = 0; l != NumElts; l += 16) {
13915 for (
unsigned i = 0; i != 16; ++i) {
13916 unsigned Idx = i + ShiftVal;
13917 if (Idx >= 16) Idx += NumElts - 16;
13918 Indices[l + i] = Idx + l;
13922 auto *VecTy = llvm::FixedVectorType::get(
Int8Ty, NumElts);
13923 Value *
Cast = Builder.CreateBitCast(Ops[0], VecTy,
"cast");
13924 Value *
Zero = llvm::Constant::getNullValue(VecTy);
13925 Value *SV = Builder.CreateShuffleVector(Cast, Zero,
13926 makeArrayRef(Indices, NumElts),
13928 return Builder.CreateBitCast(SV, ResultType,
"cast");
13930 case X86::BI__builtin_ia32_kshiftliqi:
13931 case X86::BI__builtin_ia32_kshiftlihi:
13932 case X86::BI__builtin_ia32_kshiftlisi:
13933 case X86::BI__builtin_ia32_kshiftlidi: {
13935 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
13937 if (ShiftVal >= NumElts)
13938 return llvm::Constant::getNullValue(Ops[0]->
getType());
13943 for (
unsigned i = 0; i != NumElts; ++i)
13944 Indices[i] = NumElts + i - ShiftVal;
13946 Value *
Zero = llvm::Constant::getNullValue(In->getType());
13947 Value *SV = Builder.CreateShuffleVector(Zero, In,
13948 makeArrayRef(Indices, NumElts),
13950 return Builder.CreateBitCast(SV, Ops[0]->
getType());
13952 case X86::BI__builtin_ia32_kshiftriqi:
13953 case X86::BI__builtin_ia32_kshiftrihi:
13954 case X86::BI__builtin_ia32_kshiftrisi:
13955 case X86::BI__builtin_ia32_kshiftridi: {
13957 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
13959 if (ShiftVal >= NumElts)
13960 return llvm::Constant::getNullValue(Ops[0]->
getType());
13965 for (
unsigned i = 0; i != NumElts; ++i)
13966 Indices[i] = i + ShiftVal;
13968 Value *
Zero = llvm::Constant::getNullValue(In->getType());
13969 Value *SV = Builder.CreateShuffleVector(In, Zero,
13970 makeArrayRef(Indices, NumElts),
13972 return Builder.CreateBitCast(SV, Ops[0]->
getType());
13974 case X86::BI__builtin_ia32_movnti:
13975 case X86::BI__builtin_ia32_movnti64:
13976 case X86::BI__builtin_ia32_movntsd:
13977 case X86::BI__builtin_ia32_movntss: {
13978 llvm::MDNode *
Node = llvm::MDNode::get(
13979 getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
13981 Value *Ptr = Ops[0];
13982 Value *Src = Ops[1];
13985 if (BuiltinID == X86::BI__builtin_ia32_movntsd ||
13986 BuiltinID == X86::BI__builtin_ia32_movntss)
13987 Src = Builder.CreateExtractElement(Src, (
uint64_t)0,
"extract");
13990 Value *BC = Builder.CreateBitCast(
13991 Ptr, llvm::PointerType::getUnqual(Src->getType()),
"cast");
13994 StoreInst *SI = Builder.CreateDefaultAlignedStore(Src, BC);
13996 SI->setAlignment(llvm::Align(1));
14000 case X86::BI__builtin_ia32_vprotb:
14001 case X86::BI__builtin_ia32_vprotw:
14002 case X86::BI__builtin_ia32_vprotd:
14003 case X86::BI__builtin_ia32_vprotq:
14004 case X86::BI__builtin_ia32_vprotbi:
14005 case X86::BI__builtin_ia32_vprotwi:
14006 case X86::BI__builtin_ia32_vprotdi:
14007 case X86::BI__builtin_ia32_vprotqi:
14008 case X86::BI__builtin_ia32_prold128:
14009 case X86::BI__builtin_ia32_prold256:
14010 case X86::BI__builtin_ia32_prold512:
14011 case X86::BI__builtin_ia32_prolq128:
14012 case X86::BI__builtin_ia32_prolq256:
14013 case X86::BI__builtin_ia32_prolq512:
14014 case X86::BI__builtin_ia32_prolvd128:
14015 case X86::BI__builtin_ia32_prolvd256:
14016 case X86::BI__builtin_ia32_prolvd512:
14017 case X86::BI__builtin_ia32_prolvq128:
14018 case X86::BI__builtin_ia32_prolvq256:
14019 case X86::BI__builtin_ia32_prolvq512:
14021 case X86::BI__builtin_ia32_prord128:
14022 case X86::BI__builtin_ia32_prord256:
14023 case X86::BI__builtin_ia32_prord512:
14024 case X86::BI__builtin_ia32_prorq128:
14025 case X86::BI__builtin_ia32_prorq256:
14026 case X86::BI__builtin_ia32_prorq512:
14027 case X86::BI__builtin_ia32_prorvd128:
14028 case X86::BI__builtin_ia32_prorvd256:
14029 case X86::BI__builtin_ia32_prorvd512:
14030 case X86::BI__builtin_ia32_prorvq128:
14031 case X86::BI__builtin_ia32_prorvq256:
14032 case X86::BI__builtin_ia32_prorvq512:
14034 case X86::BI__builtin_ia32_selectb_128:
14035 case X86::BI__builtin_ia32_selectb_256:
14036 case X86::BI__builtin_ia32_selectb_512:
14037 case X86::BI__builtin_ia32_selectw_128:
14038 case X86::BI__builtin_ia32_selectw_256:
14039 case X86::BI__builtin_ia32_selectw_512:
14040 case X86::BI__builtin_ia32_selectd_128:
14041 case X86::BI__builtin_ia32_selectd_256:
14042 case X86::BI__builtin_ia32_selectd_512:
14043 case X86::BI__builtin_ia32_selectq_128:
14044 case X86::BI__builtin_ia32_selectq_256:
14045 case X86::BI__builtin_ia32_selectq_512:
14046 case X86::BI__builtin_ia32_selectph_128:
14047 case X86::BI__builtin_ia32_selectph_256:
14048 case X86::BI__builtin_ia32_selectph_512:
14049 case X86::BI__builtin_ia32_selectps_128:
14050 case X86::BI__builtin_ia32_selectps_256:
14051 case X86::BI__builtin_ia32_selectps_512:
14052 case X86::BI__builtin_ia32_selectpd_128:
14053 case X86::BI__builtin_ia32_selectpd_256:
14054 case X86::BI__builtin_ia32_selectpd_512:
14056 case X86::BI__builtin_ia32_selectsh_128:
14057 case X86::BI__builtin_ia32_selectss_128:
14058 case X86::BI__builtin_ia32_selectsd_128: {
14059 Value *A = Builder.CreateExtractElement(Ops[1], (
uint64_t)0);
14060 Value *B = Builder.CreateExtractElement(Ops[2], (
uint64_t)0);
14062 return Builder.CreateInsertElement(Ops[1], A, (
uint64_t)0);
14064 case X86::BI__builtin_ia32_cmpb128_mask:
14065 case X86::BI__builtin_ia32_cmpb256_mask:
14066 case X86::BI__builtin_ia32_cmpb512_mask:
14067 case X86::BI__builtin_ia32_cmpw128_mask:
14068 case X86::BI__builtin_ia32_cmpw256_mask:
14069 case X86::BI__builtin_ia32_cmpw512_mask:
14070 case X86::BI__builtin_ia32_cmpd128_mask:
14071 case X86::BI__builtin_ia32_cmpd256_mask:
14072 case X86::BI__builtin_ia32_cmpd512_mask:
14073 case X86::BI__builtin_ia32_cmpq128_mask:
14074 case X86::BI__builtin_ia32_cmpq256_mask:
14075 case X86::BI__builtin_ia32_cmpq512_mask: {
14079 case X86::BI__builtin_ia32_ucmpb128_mask:
14080 case X86::BI__builtin_ia32_ucmpb256_mask:
14081 case X86::BI__builtin_ia32_ucmpb512_mask:
14082 case X86::BI__builtin_ia32_ucmpw128_mask:
14083 case X86::BI__builtin_ia32_ucmpw256_mask:
14084 case X86::BI__builtin_ia32_ucmpw512_mask:
14085 case X86::BI__builtin_ia32_ucmpd128_mask:
14086 case X86::BI__builtin_ia32_ucmpd256_mask:
14087 case X86::BI__builtin_ia32_ucmpd512_mask:
14088 case X86::BI__builtin_ia32_ucmpq128_mask:
14089 case X86::BI__builtin_ia32_ucmpq256_mask:
14090 case X86::BI__builtin_ia32_ucmpq512_mask: {
14094 case X86::BI__builtin_ia32_vpcomb:
14095 case X86::BI__builtin_ia32_vpcomw:
14096 case X86::BI__builtin_ia32_vpcomd:
14097 case X86::BI__builtin_ia32_vpcomq:
14099 case X86::BI__builtin_ia32_vpcomub:
14100 case X86::BI__builtin_ia32_vpcomuw:
14101 case X86::BI__builtin_ia32_vpcomud:
14102 case X86::BI__builtin_ia32_vpcomuq:
14105 case X86::BI__builtin_ia32_kortestcqi:
14106 case X86::BI__builtin_ia32_kortestchi:
14107 case X86::BI__builtin_ia32_kortestcsi:
14108 case X86::BI__builtin_ia32_kortestcdi: {
14110 Value *
C = llvm::Constant::getAllOnesValue(Ops[0]->
getType());
14111 Value *Cmp = Builder.CreateICmpEQ(Or,
C);
14114 case X86::BI__builtin_ia32_kortestzqi:
14115 case X86::BI__builtin_ia32_kortestzhi:
14116 case X86::BI__builtin_ia32_kortestzsi:
14117 case X86::BI__builtin_ia32_kortestzdi: {
14119 Value *
C = llvm::Constant::getNullValue(Ops[0]->
getType());
14120 Value *Cmp = Builder.CreateICmpEQ(Or,
C);
14124 case X86::BI__builtin_ia32_ktestcqi:
14125 case X86::BI__builtin_ia32_ktestzqi:
14126 case X86::BI__builtin_ia32_ktestchi:
14127 case X86::BI__builtin_ia32_ktestzhi:
14128 case X86::BI__builtin_ia32_ktestcsi:
14129 case X86::BI__builtin_ia32_ktestzsi:
14130 case X86::BI__builtin_ia32_ktestcdi:
14131 case X86::BI__builtin_ia32_ktestzdi: {
14133 switch (BuiltinID) {
14134 default: llvm_unreachable(
"Unsupported intrinsic!");
14135 case X86::BI__builtin_ia32_ktestcqi:
14136 IID = Intrinsic::x86_avx512_ktestc_b;
14138 case X86::BI__builtin_ia32_ktestzqi:
14139 IID = Intrinsic::x86_avx512_ktestz_b;
14141 case X86::BI__builtin_ia32_ktestchi:
14142 IID = Intrinsic::x86_avx512_ktestc_w;
14144 case X86::BI__builtin_ia32_ktestzhi:
14145 IID = Intrinsic::x86_avx512_ktestz_w;
14147 case X86::BI__builtin_ia32_ktestcsi:
14148 IID = Intrinsic::x86_avx512_ktestc_d;
14150 case X86::BI__builtin_ia32_ktestzsi:
14151 IID = Intrinsic::x86_avx512_ktestz_d;
14153 case X86::BI__builtin_ia32_ktestcdi:
14154 IID = Intrinsic::x86_avx512_ktestc_q;
14156 case X86::BI__builtin_ia32_ktestzdi:
14157 IID = Intrinsic::x86_avx512_ktestz_q;
14161 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
14165 return Builder.CreateCall(Intr, {LHS, RHS});
14168 case X86::BI__builtin_ia32_kaddqi:
14169 case X86::BI__builtin_ia32_kaddhi:
14170 case X86::BI__builtin_ia32_kaddsi:
14171 case X86::BI__builtin_ia32_kadddi: {
14173 switch (BuiltinID) {
14174 default: llvm_unreachable(
"Unsupported intrinsic!");
14175 case X86::BI__builtin_ia32_kaddqi:
14176 IID = Intrinsic::x86_avx512_kadd_b;
14178 case X86::BI__builtin_ia32_kaddhi:
14179 IID = Intrinsic::x86_avx512_kadd_w;
14181 case X86::BI__builtin_ia32_kaddsi:
14182 IID = Intrinsic::x86_avx512_kadd_d;
14184 case X86::BI__builtin_ia32_kadddi:
14185 IID = Intrinsic::x86_avx512_kadd_q;
14189 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
14193 Value *Res = Builder.CreateCall(Intr, {LHS, RHS});
14194 return Builder.CreateBitCast(Res, Ops[0]->
getType());
14196 case X86::BI__builtin_ia32_kandqi:
14197 case X86::BI__builtin_ia32_kandhi:
14198 case X86::BI__builtin_ia32_kandsi:
14199 case X86::BI__builtin_ia32_kanddi:
14201 case X86::BI__builtin_ia32_kandnqi:
14202 case X86::BI__builtin_ia32_kandnhi:
14203 case X86::BI__builtin_ia32_kandnsi:
14204 case X86::BI__builtin_ia32_kandndi:
14206 case X86::BI__builtin_ia32_korqi:
14207 case X86::BI__builtin_ia32_korhi:
14208 case X86::BI__builtin_ia32_korsi:
14209 case X86::BI__builtin_ia32_kordi:
14211 case X86::BI__builtin_ia32_kxnorqi:
14212 case X86::BI__builtin_ia32_kxnorhi:
14213 case X86::BI__builtin_ia32_kxnorsi:
14214 case X86::BI__builtin_ia32_kxnordi:
14216 case X86::BI__builtin_ia32_kxorqi:
14217 case X86::BI__builtin_ia32_kxorhi:
14218 case X86::BI__builtin_ia32_kxorsi:
14219 case X86::BI__builtin_ia32_kxordi:
14221 case X86::BI__builtin_ia32_knotqi:
14222 case X86::BI__builtin_ia32_knothi:
14223 case X86::BI__builtin_ia32_knotsi:
14224 case X86::BI__builtin_ia32_knotdi: {
14225 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
14227 return Builder.CreateBitCast(Builder.CreateNot(Res),
14228 Ops[0]->getType());
14230 case X86::BI__builtin_ia32_kmovb:
14231 case X86::BI__builtin_ia32_kmovw:
14232 case X86::BI__builtin_ia32_kmovd:
14233 case X86::BI__builtin_ia32_kmovq: {
14237 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
14239 return Builder.CreateBitCast(Res, Ops[0]->
getType());
14242 case X86::BI__builtin_ia32_kunpckdi:
14243 case X86::BI__builtin_ia32_kunpcksi:
14244 case X86::BI__builtin_ia32_kunpckhi: {
14245 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
14249 for (
unsigned i = 0; i != NumElts; ++i)
14254 LHS = Builder.CreateShuffleVector(LHS, LHS,
14255 makeArrayRef(Indices, NumElts / 2));
14256 RHS = Builder.CreateShuffleVector(RHS, RHS,
14257 makeArrayRef(Indices, NumElts / 2));
14260 Value *Res = Builder.CreateShuffleVector(RHS, LHS,
14261 makeArrayRef(Indices, NumElts));
14262 return Builder.CreateBitCast(Res, Ops[0]->
getType());
14265 case X86::BI__builtin_ia32_vplzcntd_128:
14266 case X86::BI__builtin_ia32_vplzcntd_256:
14267 case X86::BI__builtin_ia32_vplzcntd_512:
14268 case X86::BI__builtin_ia32_vplzcntq_128:
14269 case X86::BI__builtin_ia32_vplzcntq_256:
14270 case X86::BI__builtin_ia32_vplzcntq_512: {
14272 return Builder.CreateCall(F, {Ops[0],Builder.getInt1(
false)});
14274 case X86::BI__builtin_ia32_sqrtss:
14275 case X86::BI__builtin_ia32_sqrtsd: {
14276 Value *A = Builder.CreateExtractElement(Ops[0], (
uint64_t)0);
14278 if (Builder.getIsFPConstrained()) {
14279 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
14282 A = Builder.CreateConstrainedFPCall(F, {A});
14285 A = Builder.CreateCall(F, {A});
14287 return Builder.CreateInsertElement(Ops[0], A, (
uint64_t)0);
14289 case X86::BI__builtin_ia32_sqrtsh_round_mask:
14290 case X86::BI__builtin_ia32_sqrtsd_round_mask:
14291 case X86::BI__builtin_ia32_sqrtss_round_mask: {
14298 switch (BuiltinID) {
14300 llvm_unreachable(
"Unsupported intrinsic!");
14301 case X86::BI__builtin_ia32_sqrtsh_round_mask:
14302 IID = Intrinsic::x86_avx512fp16_mask_sqrt_sh;
14304 case X86::BI__builtin_ia32_sqrtsd_round_mask:
14305 IID = Intrinsic::x86_avx512_mask_sqrt_sd;
14307 case X86::BI__builtin_ia32_sqrtss_round_mask:
14308 IID = Intrinsic::x86_avx512_mask_sqrt_ss;
14313 Value *A = Builder.CreateExtractElement(Ops[1], (
uint64_t)0);
14315 if (Builder.getIsFPConstrained()) {
14316 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
14319 A = Builder.CreateConstrainedFPCall(F, A);
14322 A = Builder.CreateCall(F, A);
14324 Value *Src = Builder.CreateExtractElement(Ops[2], (
uint64_t)0);
14326 return Builder.CreateInsertElement(Ops[0], A, (
uint64_t)0);
14328 case X86::BI__builtin_ia32_sqrtpd256:
14329 case X86::BI__builtin_ia32_sqrtpd:
14330 case X86::BI__builtin_ia32_sqrtps256:
14331 case X86::BI__builtin_ia32_sqrtps:
14332 case X86::BI__builtin_ia32_sqrtph256:
14333 case X86::BI__builtin_ia32_sqrtph:
14334 case X86::BI__builtin_ia32_sqrtph512:
14335 case X86::BI__builtin_ia32_sqrtps512:
14336 case X86::BI__builtin_ia32_sqrtpd512: {
14337 if (Ops.size() == 2) {
14344 switch (BuiltinID) {
14346 llvm_unreachable(
"Unsupported intrinsic!");
14347 case X86::BI__builtin_ia32_sqrtph512:
14348 IID = Intrinsic::x86_avx512fp16_sqrt_ph_512;
14350 case X86::BI__builtin_ia32_sqrtps512:
14351 IID = Intrinsic::x86_avx512_sqrt_ps_512;
14353 case X86::BI__builtin_ia32_sqrtpd512:
14354 IID = Intrinsic::x86_avx512_sqrt_pd_512;
14360 if (Builder.getIsFPConstrained()) {
14361 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
14362 Function *F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_sqrt,
14364 return Builder.CreateConstrainedFPCall(F, Ops[0]);
14367 return Builder.CreateCall(F, Ops[0]);
14371 case X86::BI__builtin_ia32_pmuludq128:
14372 case X86::BI__builtin_ia32_pmuludq256:
14373 case X86::BI__builtin_ia32_pmuludq512:
14376 case X86::BI__builtin_ia32_pmuldq128:
14377 case X86::BI__builtin_ia32_pmuldq256:
14378 case X86::BI__builtin_ia32_pmuldq512:
14381 case X86::BI__builtin_ia32_pternlogd512_mask:
14382 case X86::BI__builtin_ia32_pternlogq512_mask:
14383 case X86::BI__builtin_ia32_pternlogd128_mask:
14384 case X86::BI__builtin_ia32_pternlogd256_mask:
14385 case X86::BI__builtin_ia32_pternlogq128_mask:
14386 case X86::BI__builtin_ia32_pternlogq256_mask:
14389 case X86::BI__builtin_ia32_pternlogd512_maskz:
14390 case X86::BI__builtin_ia32_pternlogq512_maskz:
14391 case X86::BI__builtin_ia32_pternlogd128_maskz:
14392 case X86::BI__builtin_ia32_pternlogd256_maskz:
14393 case X86::BI__builtin_ia32_pternlogq128_maskz:
14394 case X86::BI__builtin_ia32_pternlogq256_maskz:
14397 case X86::BI__builtin_ia32_vpshldd128:
14398 case X86::BI__builtin_ia32_vpshldd256:
14399 case X86::BI__builtin_ia32_vpshldd512:
14400 case X86::BI__builtin_ia32_vpshldq128:
14401 case X86::BI__builtin_ia32_vpshldq256:
14402 case X86::BI__builtin_ia32_vpshldq512:
14403 case X86::BI__builtin_ia32_vpshldw128:
14404 case X86::BI__builtin_ia32_vpshldw256:
14405 case X86::BI__builtin_ia32_vpshldw512:
14408 case X86::BI__builtin_ia32_vpshrdd128:
14409 case X86::BI__builtin_ia32_vpshrdd256:
14410 case X86::BI__builtin_ia32_vpshrdd512:
14411 case X86::BI__builtin_ia32_vpshrdq128:
14412 case X86::BI__builtin_ia32_vpshrdq256:
14413 case X86::BI__builtin_ia32_vpshrdq512:
14414 case X86::BI__builtin_ia32_vpshrdw128:
14415 case X86::BI__builtin_ia32_vpshrdw256:
14416 case X86::BI__builtin_ia32_vpshrdw512:
14420 case X86::BI__builtin_ia32_vpshldvd128:
14421 case X86::BI__builtin_ia32_vpshldvd256:
14422 case X86::BI__builtin_ia32_vpshldvd512:
14423 case X86::BI__builtin_ia32_vpshldvq128:
14424 case X86::BI__builtin_ia32_vpshldvq256:
14425 case X86::BI__builtin_ia32_vpshldvq512:
14426 case X86::BI__builtin_ia32_vpshldvw128:
14427 case X86::BI__builtin_ia32_vpshldvw256:
14428 case X86::BI__builtin_ia32_vpshldvw512:
14431 case X86::BI__builtin_ia32_vpshrdvd128:
14432 case X86::BI__builtin_ia32_vpshrdvd256:
14433 case X86::BI__builtin_ia32_vpshrdvd512:
14434 case X86::BI__builtin_ia32_vpshrdvq128:
14435 case X86::BI__builtin_ia32_vpshrdvq256:
14436 case X86::BI__builtin_ia32_vpshrdvq512:
14437 case X86::BI__builtin_ia32_vpshrdvw128:
14438 case X86::BI__builtin_ia32_vpshrdvw256:
14439 case X86::BI__builtin_ia32_vpshrdvw512:
14444 case X86::BI__builtin_ia32_reduce_fadd_pd512:
14445 case X86::BI__builtin_ia32_reduce_fadd_ps512:
14446 case X86::BI__builtin_ia32_reduce_fadd_ph512:
14447 case X86::BI__builtin_ia32_reduce_fadd_ph256:
14448 case X86::BI__builtin_ia32_reduce_fadd_ph128: {
14451 Builder.getFastMathFlags().setAllowReassoc();
14452 return Builder.CreateCall(F, {Ops[0], Ops[1]});
14454 case X86::BI__builtin_ia32_reduce_fmul_pd512:
14455 case X86::BI__builtin_ia32_reduce_fmul_ps512:
14456 case X86::BI__builtin_ia32_reduce_fmul_ph512:
14457 case X86::BI__builtin_ia32_reduce_fmul_ph256:
14458 case X86::BI__builtin_ia32_reduce_fmul_ph128: {
14461 Builder.getFastMathFlags().setAllowReassoc();
14462 return Builder.CreateCall(F, {Ops[0], Ops[1]});
14464 case X86::BI__builtin_ia32_reduce_fmax_pd512:
14465 case X86::BI__builtin_ia32_reduce_fmax_ps512:
14466 case X86::BI__builtin_ia32_reduce_fmax_ph512:
14467 case X86::BI__builtin_ia32_reduce_fmax_ph256:
14468 case X86::BI__builtin_ia32_reduce_fmax_ph128: {
14471 Builder.getFastMathFlags().setNoNaNs();
14472 return Builder.CreateCall(F, {Ops[0]});
14474 case X86::BI__builtin_ia32_reduce_fmin_pd512:
14475 case X86::BI__builtin_ia32_reduce_fmin_ps512:
14476 case X86::BI__builtin_ia32_reduce_fmin_ph512:
14477 case X86::BI__builtin_ia32_reduce_fmin_ph256:
14478 case X86::BI__builtin_ia32_reduce_fmin_ph128: {
14481 Builder.getFastMathFlags().setNoNaNs();
14482 return Builder.CreateCall(F, {Ops[0]});
14486 case X86::BI__builtin_ia32_pswapdsf:
14487 case X86::BI__builtin_ia32_pswapdsi: {
14489 Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy,
"cast");
14491 return Builder.CreateCall(F, Ops,
"pswapd");
14493 case X86::BI__builtin_ia32_rdrand16_step:
14494 case X86::BI__builtin_ia32_rdrand32_step:
14495 case X86::BI__builtin_ia32_rdrand64_step:
14496 case X86::BI__builtin_ia32_rdseed16_step:
14497 case X86::BI__builtin_ia32_rdseed32_step:
14498 case X86::BI__builtin_ia32_rdseed64_step: {
14500 switch (BuiltinID) {
14501 default: llvm_unreachable(
"Unsupported intrinsic!");
14502 case X86::BI__builtin_ia32_rdrand16_step:
14503 ID = Intrinsic::x86_rdrand_16;
14505 case X86::BI__builtin_ia32_rdrand32_step:
14506 ID = Intrinsic::x86_rdrand_32;
14508 case X86::BI__builtin_ia32_rdrand64_step:
14509 ID = Intrinsic::x86_rdrand_64;
14511 case X86::BI__builtin_ia32_rdseed16_step:
14512 ID = Intrinsic::x86_rdseed_16;
14514 case X86::BI__builtin_ia32_rdseed32_step:
14515 ID = Intrinsic::x86_rdseed_32;
14517 case X86::BI__builtin_ia32_rdseed64_step:
14518 ID = Intrinsic::x86_rdseed_64;
14523 Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 0),
14525 return Builder.CreateExtractValue(Call, 1);
14527 case X86::BI__builtin_ia32_addcarryx_u32:
14528 case X86::BI__builtin_ia32_addcarryx_u64:
14529 case X86::BI__builtin_ia32_subborrow_u32:
14530 case X86::BI__builtin_ia32_subborrow_u64: {
14532 switch (BuiltinID) {
14533 default: llvm_unreachable(
"Unsupported intrinsic!");
14534 case X86::BI__builtin_ia32_addcarryx_u32:
14535 IID = Intrinsic::x86_addcarry_32;
14537 case X86::BI__builtin_ia32_addcarryx_u64:
14538 IID = Intrinsic::x86_addcarry_64;
14540 case X86::BI__builtin_ia32_subborrow_u32:
14541 IID = Intrinsic::x86_subborrow_32;
14543 case X86::BI__builtin_ia32_subborrow_u64:
14544 IID = Intrinsic::x86_subborrow_64;
14549 { Ops[0], Ops[1], Ops[2] });
14550 Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 1),
14552 return Builder.CreateExtractValue(Call, 0);
14555 case X86::BI__builtin_ia32_fpclassps128_mask:
14556 case X86::BI__builtin_ia32_fpclassps256_mask:
14557 case X86::BI__builtin_ia32_fpclassps512_mask:
14558 case X86::BI__builtin_ia32_fpclassph128_mask:
14559 case X86::BI__builtin_ia32_fpclassph256_mask:
14560 case X86::BI__builtin_ia32_fpclassph512_mask:
14561 case X86::BI__builtin_ia32_fpclasspd128_mask:
14562 case X86::BI__builtin_ia32_fpclasspd256_mask:
14563 case X86::BI__builtin_ia32_fpclasspd512_mask: {
14566 Value *MaskIn = Ops[2];
14567 Ops.erase(&Ops[2]);
14570 switch (BuiltinID) {
14571 default: llvm_unreachable(
"Unsupported intrinsic!");
14572 case X86::BI__builtin_ia32_fpclassph128_mask:
14573 ID = Intrinsic::x86_avx512fp16_fpclass_ph_128;
14575 case X86::BI__builtin_ia32_fpclassph256_mask:
14576 ID = Intrinsic::x86_avx512fp16_fpclass_ph_256;
14578 case X86::BI__builtin_ia32_fpclassph512_mask:
14579 ID = Intrinsic::x86_avx512fp16_fpclass_ph_512;
14581 case X86::BI__builtin_ia32_fpclassps128_mask:
14582 ID = Intrinsic::x86_avx512_fpclass_ps_128;
14584 case X86::BI__builtin_ia32_fpclassps256_mask:
14585 ID = Intrinsic::x86_avx512_fpclass_ps_256;
14587 case X86::BI__builtin_ia32_fpclassps512_mask:
14588 ID = Intrinsic::x86_avx512_fpclass_ps_512;
14590 case X86::BI__builtin_ia32_fpclasspd128_mask:
14591 ID = Intrinsic::x86_avx512_fpclass_pd_128;
14593 case X86::BI__builtin_ia32_fpclasspd256_mask:
14594 ID = Intrinsic::x86_avx512_fpclass_pd_256;
14596 case X86::BI__builtin_ia32_fpclasspd512_mask:
14597 ID = Intrinsic::x86_avx512_fpclass_pd_512;
14605 case X86::BI__builtin_ia32_vp2intersect_q_512:
14606 case X86::BI__builtin_ia32_vp2intersect_q_256:
14607 case X86::BI__builtin_ia32_vp2intersect_q_128:
14608 case X86::BI__builtin_ia32_vp2intersect_d_512:
14609 case X86::BI__builtin_ia32_vp2intersect_d_256:
14610 case X86::BI__builtin_ia32_vp2intersect_d_128: {
14615 switch (BuiltinID) {
14616 default: llvm_unreachable(
"Unsupported intrinsic!");
14617 case X86::BI__builtin_ia32_vp2intersect_q_512:
14618 ID = Intrinsic::x86_avx512_vp2intersect_q_512;
14620 case X86::BI__builtin_ia32_vp2intersect_q_256:
14621 ID = Intrinsic::x86_avx512_vp2intersect_q_256;
14623 case X86::BI__builtin_ia32_vp2intersect_q_128:
14624 ID = Intrinsic::x86_avx512_vp2intersect_q_128;
14626 case X86::BI__builtin_ia32_vp2intersect_d_512:
14627 ID = Intrinsic::x86_avx512_vp2intersect_d_512;
14629 case X86::BI__builtin_ia32_vp2intersect_d_256:
14630 ID = Intrinsic::x86_avx512_vp2intersect_d_256;
14632 case X86::BI__builtin_ia32_vp2intersect_d_128:
14633 ID = Intrinsic::x86_avx512_vp2intersect_d_128;
14638 Value *
Result = Builder.CreateExtractValue(Call, 0);
14640 Builder.CreateDefaultAlignedStore(
Result, Ops[2]);
14642 Result = Builder.CreateExtractValue(Call, 1);
14644 return Builder.CreateDefaultAlignedStore(
Result, Ops[3]);
14647 case X86::BI__builtin_ia32_vpmultishiftqb128:
14648 case X86::BI__builtin_ia32_vpmultishiftqb256:
14649 case X86::BI__builtin_ia32_vpmultishiftqb512: {
14651 switch (BuiltinID) {
14652 default: llvm_unreachable(
"Unsupported intrinsic!");
14653 case X86::BI__builtin_ia32_vpmultishiftqb128:
14654 ID = Intrinsic::x86_avx512_pmultishift_qb_128;
14656 case X86::BI__builtin_ia32_vpmultishiftqb256:
14657 ID = Intrinsic::x86_avx512_pmultishift_qb_256;
14659 case X86::BI__builtin_ia32_vpmultishiftqb512:
14660 ID = Intrinsic::x86_avx512_pmultishift_qb_512;
14667 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
14668 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
14669 case X86::BI__builtin_ia32_vpshufbitqmb512_mask: {
14672 Value *MaskIn = Ops[2];
14673 Ops.erase(&Ops[2]);
14676 switch (BuiltinID) {
14677 default: llvm_unreachable(
"Unsupported intrinsic!");
14678 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
14679 ID = Intrinsic::x86_avx512_vpshufbitqmb_128;
14681 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
14682 ID = Intrinsic::x86_avx512_vpshufbitqmb_256;
14684 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
14685 ID = Intrinsic::x86_avx512_vpshufbitqmb_512;
14694 case X86::BI__builtin_ia32_cmpeqps:
14695 case X86::BI__builtin_ia32_cmpeqpd:
14696 return getVectorFCmpIR(CmpInst::FCMP_OEQ,
false);
14697 case X86::BI__builtin_ia32_cmpltps:
14698 case X86::BI__builtin_ia32_cmpltpd:
14699 return getVectorFCmpIR(CmpInst::FCMP_OLT,
true);
14700 case X86::BI__builtin_ia32_cmpleps:
14701 case X86::BI__builtin_ia32_cmplepd:
14702 return getVectorFCmpIR(CmpInst::FCMP_OLE,
true);
14703 case X86::BI__builtin_ia32_cmpunordps:
14704 case X86::BI__builtin_ia32_cmpunordpd:
14705 return getVectorFCmpIR(CmpInst::FCMP_UNO,
false);
14706 case X86::BI__builtin_ia32_cmpneqps:
14707 case X86::BI__builtin_ia32_cmpneqpd:
14708 return getVectorFCmpIR(CmpInst::FCMP_UNE,
false);
14709 case X86::BI__builtin_ia32_cmpnltps:
14710 case X86::BI__builtin_ia32_cmpnltpd:
14711 return getVectorFCmpIR(CmpInst::FCMP_UGE,
true);
14712 case X86::BI__builtin_ia32_cmpnleps:
14713 case X86::BI__builtin_ia32_cmpnlepd:
14714 return getVectorFCmpIR(CmpInst::FCMP_UGT,
true);
14715 case X86::BI__builtin_ia32_cmpordps:
14716 case X86::BI__builtin_ia32_cmpordpd:
14717 return getVectorFCmpIR(CmpInst::FCMP_ORD,
false);
14718 case X86::BI__builtin_ia32_cmpph128_mask:
14719 case X86::BI__builtin_ia32_cmpph256_mask:
14720 case X86::BI__builtin_ia32_cmpph512_mask:
14721 case X86::BI__builtin_ia32_cmpps128_mask:
14722 case X86::BI__builtin_ia32_cmpps256_mask:
14723 case X86::BI__builtin_ia32_cmpps512_mask:
14724 case X86::BI__builtin_ia32_cmppd128_mask:
14725 case X86::BI__builtin_ia32_cmppd256_mask:
14726 case X86::BI__builtin_ia32_cmppd512_mask:
14729 case X86::BI__builtin_ia32_cmpps:
14730 case X86::BI__builtin_ia32_cmpps256:
14731 case X86::BI__builtin_ia32_cmppd:
14732 case X86::BI__builtin_ia32_cmppd256: {
14745 FCmpInst::Predicate Pred;
14749 switch (CC & 0xf) {
14750 case 0x00: Pred = FCmpInst::FCMP_OEQ; IsSignaling =
false;
break;
14751 case 0x01: Pred = FCmpInst::FCMP_OLT; IsSignaling =
true;
break;
14752 case 0x02: Pred = FCmpInst::FCMP_OLE; IsSignaling =
true;
break;
14753 case 0x03: Pred = FCmpInst::FCMP_UNO; IsSignaling =
false;
break;
14754 case 0x04: Pred = FCmpInst::FCMP_UNE; IsSignaling =
false;
break;
14755 case 0x05: Pred = FCmpInst::FCMP_UGE; IsSignaling =
true;
break;
14756 case 0x06: Pred = FCmpInst::FCMP_UGT; IsSignaling =
true;
break;
14757 case 0x07: Pred = FCmpInst::FCMP_ORD; IsSignaling =
false;
break;
14758 case 0x08: Pred = FCmpInst::FCMP_UEQ; IsSignaling =
false;
break;
14759 case 0x09: Pred = FCmpInst::FCMP_ULT; IsSignaling =
true;
break;
14760 case 0x0a: Pred = FCmpInst::FCMP_ULE; IsSignaling =
true;
break;
14761 case 0x0b: Pred = FCmpInst::FCMP_FALSE; IsSignaling =
false;
break;
14762 case 0x0c: Pred = FCmpInst::FCMP_ONE; IsSignaling =
false;
break;
14763 case 0x0d: Pred = FCmpInst::FCMP_OGE; IsSignaling =
true;
break;
14764 case 0x0e: Pred = FCmpInst::FCMP_OGT; IsSignaling =
true;
break;
14765 case 0x0f: Pred = FCmpInst::FCMP_TRUE; IsSignaling =
false;
break;
14766 default: llvm_unreachable(
"Unhandled CC");
14771 IsSignaling = !IsSignaling;
14778 if (Builder.getIsFPConstrained() &&
14779 (Pred == FCmpInst::FCMP_TRUE || Pred == FCmpInst::FCMP_FALSE ||
14783 switch (BuiltinID) {
14784 default: llvm_unreachable(
"Unexpected builtin");
14785 case X86::BI__builtin_ia32_cmpps:
14786 IID = Intrinsic::x86_sse_cmp_ps;
14788 case X86::BI__builtin_ia32_cmpps256:
14789 IID = Intrinsic::x86_avx_cmp_ps_256;
14791 case X86::BI__builtin_ia32_cmppd:
14792 IID = Intrinsic::x86_sse2_cmp_pd;
14794 case X86::BI__builtin_ia32_cmppd256:
14795 IID = Intrinsic::x86_avx_cmp_pd_256;
14797 case X86::BI__builtin_ia32_cmpps512_mask:
14798 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
14800 case X86::BI__builtin_ia32_cmppd512_mask:
14801 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
14803 case X86::BI__builtin_ia32_cmpps128_mask:
14804 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
14806 case X86::BI__builtin_ia32_cmpps256_mask:
14807 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
14809 case X86::BI__builtin_ia32_cmppd128_mask:
14810 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
14812 case X86::BI__builtin_ia32_cmppd256_mask:
14813 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
14822 Value *Cmp = Builder.CreateCall(Intr, Ops);
14826 return Builder.CreateCall(Intr, Ops);
14840 Cmp = Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
14842 Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
14846 return getVectorFCmpIR(Pred, IsSignaling);
14850 case X86::BI__builtin_ia32_cmpeqss:
14851 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 0);
14852 case X86::BI__builtin_ia32_cmpltss:
14853 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 1);
14854 case X86::BI__builtin_ia32_cmpless:
14855 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 2);
14856 case X86::BI__builtin_ia32_cmpunordss:
14857 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 3);
14858 case X86::BI__builtin_ia32_cmpneqss:
14859 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 4);
14860 case X86::BI__builtin_ia32_cmpnltss:
14861 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 5);
14862 case X86::BI__builtin_ia32_cmpnless:
14863 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 6);
14864 case X86::BI__builtin_ia32_cmpordss:
14865 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 7);
14866 case X86::BI__builtin_ia32_cmpeqsd:
14867 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 0);
14868 case X86::BI__builtin_ia32_cmpltsd:
14869 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 1);
14870 case X86::BI__builtin_ia32_cmplesd:
14871 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 2);
14872 case X86::BI__builtin_ia32_cmpunordsd:
14873 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 3);
14874 case X86::BI__builtin_ia32_cmpneqsd:
14875 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 4);
14876 case X86::BI__builtin_ia32_cmpnltsd:
14877 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 5);
14878 case X86::BI__builtin_ia32_cmpnlesd:
14879 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6);
14880 case X86::BI__builtin_ia32_cmpordsd:
14881 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
14884 case X86::BI__builtin_ia32_vcvtph2ps:
14885 case X86::BI__builtin_ia32_vcvtph2ps256:
14886 case X86::BI__builtin_ia32_vcvtph2ps_mask:
14887 case X86::BI__builtin_ia32_vcvtph2ps256_mask:
14888 case X86::BI__builtin_ia32_vcvtph2ps512_mask: {
14889 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
14894 case X86::BI__builtin_ia32_cvtneps2bf16_128_mask: {
14897 cast<llvm::FixedVectorType>(Ops[0]->
getType())->getNumElements());
14898 Intrinsic::ID IID = Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128;
14901 case X86::BI__builtin_ia32_cvtsbf162ss_32:
14904 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
14905 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask: {
14907 switch (BuiltinID) {
14908 default: llvm_unreachable(
"Unsupported intrinsic!");
14909 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
14910 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_256;
14912 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask:
14913 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_512;
14920 case X86::BI__cpuid:
14921 case X86::BI__cpuidex: {
14923 Value *SubFuncId = BuiltinID == X86::BI__cpuidex
14927 llvm::StructType *CpuidRetTy =
14929 llvm::FunctionType *FTy =
14932 StringRef
Asm, Constraints;
14933 if (
getTarget().getTriple().getArch() == llvm::Triple::x86) {
14935 Constraints =
"={ax},={bx},={cx},={dx},{ax},{cx}";
14938 Asm =
"xchgq %rbx, ${1:q}\n"
14940 "xchgq %rbx, ${1:q}";
14941 Constraints =
"={ax},=r,={cx},={dx},0,2";
14944 llvm::InlineAsm *IA = llvm::InlineAsm::get(FTy,
Asm, Constraints,
14946 Value *IACall = Builder.CreateCall(IA, {FuncId, SubFuncId});
14949 for (
unsigned i = 0; i < 4; i++) {
14950 Value *Extracted = Builder.CreateExtractValue(IACall, i);
14951 Value *StorePtr = Builder.CreateConstInBoundsGEP1_32(
Int32Ty, BasePtr, i);
14960 case X86::BI__emul:
14961 case X86::BI__emulu: {
14963 bool isSigned = (BuiltinID == X86::BI__emul);
14964 Value *LHS = Builder.CreateIntCast(Ops[0],
Int64Ty, isSigned);
14965 Value *RHS = Builder.CreateIntCast(Ops[1],
Int64Ty, isSigned);
14966 return Builder.CreateMul(LHS, RHS,
"", !isSigned, isSigned);
14968 case X86::BI__mulh:
14969 case X86::BI__umulh:
14970 case X86::BI_mul128:
14971 case X86::BI_umul128: {
14973 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
14975 bool IsSigned = (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI_mul128);
14976 Value *LHS = Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned);
14977 Value *RHS = Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned);
14979 Value *MulResult, *HigherBits;
14981 MulResult = Builder.CreateNSWMul(LHS, RHS);
14982 HigherBits = Builder.CreateAShr(MulResult, 64);
14984 MulResult = Builder.CreateNUWMul(LHS, RHS);
14985 HigherBits = Builder.CreateLShr(MulResult, 64);
14987 HigherBits = Builder.CreateIntCast(HigherBits, ResType, IsSigned);
14989 if (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI__umulh)
14993 Builder.CreateStore(HigherBits, HighBitsAddress);
14994 return Builder.CreateIntCast(MulResult, ResType, IsSigned);
14997 case X86::BI__faststorefence: {
14998 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
14999 llvm::SyncScope::System);
15001 case X86::BI__shiftleft128:
15002 case X86::BI__shiftright128: {
15004 BuiltinID == X86::BI__shiftleft128 ? Intrinsic::fshl : Intrinsic::fshr,
15009 std::swap(Ops[0], Ops[1]);
15010 Ops[2] = Builder.CreateZExt(Ops[2],
Int64Ty);
15011 return Builder.CreateCall(F, Ops);
15013 case X86::BI_ReadWriteBarrier:
15014 case X86::BI_ReadBarrier:
15015 case X86::BI_WriteBarrier: {
15016 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
15017 llvm::SyncScope::SingleThread);
15020 case X86::BI_AddressOfReturnAddress: {
15023 return Builder.CreateCall(F);
15025 case X86::BI__stosb: {
15028 return Builder.CreateMemSet(Ops[0], Ops[1], Ops[2], Align(1),
true);
15033 case X86::BI__int2c: {
15035 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
false);
15036 llvm::InlineAsm *IA =
15037 llvm::InlineAsm::get(FTy,
"int $$0x2c",
"",
true);
15038 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
15040 llvm::Attribute::NoReturn);
15041 llvm::CallInst *CI = Builder.CreateCall(IA);
15042 CI->setAttributes(NoReturnAttr);
15045 case X86::BI__readfsbyte:
15046 case X86::BI__readfsword:
15047 case X86::BI__readfsdword:
15048 case X86::BI__readfsqword: {
15051 Builder.CreateIntToPtr(Ops[0], llvm::PointerType::get(
IntTy, 257));
15052 LoadInst *
Load = Builder.CreateAlignedLoad(
15054 Load->setVolatile(
true);
15057 case X86::BI__readgsbyte:
15058 case X86::BI__readgsword:
15059 case X86::BI__readgsdword:
15060 case X86::BI__readgsqword: {
15063 Builder.CreateIntToPtr(Ops[0], llvm::PointerType::get(
IntTy, 256));
15064 LoadInst *
Load = Builder.CreateAlignedLoad(
15066 Load->setVolatile(
true);
15069 case X86::BI__builtin_ia32_encodekey128_u32: {
15070 Intrinsic::ID IID = Intrinsic::x86_encodekey128;
15074 for (
int i = 0; i < 3; ++i) {
15075 Value *Extract = Builder.CreateExtractValue(Call, i + 1);
15076 Value *Ptr = Builder.CreateConstGEP1_32(
Int8Ty, Ops[2], i * 16);
15077 Ptr = Builder.CreateBitCast(
15078 Ptr, llvm::PointerType::getUnqual(Extract->getType()));
15079 Builder.CreateAlignedStore(Extract, Ptr, Align(1));
15082 return Builder.CreateExtractValue(Call, 0);
15084 case X86::BI__builtin_ia32_encodekey256_u32: {
15085 Intrinsic::ID IID = Intrinsic::x86_encodekey256;
15090 for (
int i = 0; i < 4; ++i) {
15091 Value *Extract = Builder.CreateExtractValue(Call, i + 1);
15092 Value *Ptr = Builder.CreateConstGEP1_32(
Int8Ty, Ops[3], i * 16);
15093 Ptr = Builder.CreateBitCast(
15094 Ptr, llvm::PointerType::getUnqual(Extract->getType()));
15095 Builder.CreateAlignedStore(Extract, Ptr, Align(1));
15098 return Builder.CreateExtractValue(Call, 0);
15100 case X86::BI__builtin_ia32_aesenc128kl_u8:
15101 case X86::BI__builtin_ia32_aesdec128kl_u8:
15102 case X86::BI__builtin_ia32_aesenc256kl_u8:
15103 case X86::BI__builtin_ia32_aesdec256kl_u8: {
15105 StringRef BlockName;
15106 switch (BuiltinID) {
15108 llvm_unreachable(
"Unexpected builtin");
15109 case X86::BI__builtin_ia32_aesenc128kl_u8:
15110 IID = Intrinsic::x86_aesenc128kl;
15111 BlockName =
"aesenc128kl";
15113 case X86::BI__builtin_ia32_aesdec128kl_u8:
15114 IID = Intrinsic::x86_aesdec128kl;
15115 BlockName =
"aesdec128kl";
15117 case X86::BI__builtin_ia32_aesenc256kl_u8:
15118 IID = Intrinsic::x86_aesenc256kl;
15119 BlockName =
"aesenc256kl";
15121 case X86::BI__builtin_ia32_aesdec256kl_u8:
15122 IID = Intrinsic::x86_aesdec256kl;
15123 BlockName =
"aesdec256kl";
15129 BasicBlock *NoError =
15134 Value *
Ret = Builder.CreateExtractValue(Call, 0);
15135 Value *Succ = Builder.CreateTrunc(
Ret, Builder.getInt1Ty());
15136 Value *Out = Builder.CreateExtractValue(Call, 1);
15137 Builder.CreateCondBr(Succ, NoError,
Error);
15139 Builder.SetInsertPoint(NoError);
15140 Builder.CreateDefaultAlignedStore(Out, Ops[0]);
15141 Builder.CreateBr(End);
15143 Builder.SetInsertPoint(
Error);
15144 Constant *
Zero = llvm::Constant::getNullValue(Out->getType());
15145 Builder.CreateDefaultAlignedStore(Zero, Ops[0]);
15146 Builder.CreateBr(End);
15148 Builder.SetInsertPoint(End);
15149 return Builder.CreateExtractValue(Call, 0);
15151 case X86::BI__builtin_ia32_aesencwide128kl_u8:
15152 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
15153 case X86::BI__builtin_ia32_aesencwide256kl_u8:
15154 case X86::BI__builtin_ia32_aesdecwide256kl_u8: {
15156 StringRef BlockName;
15157 switch (BuiltinID) {
15158 case X86::BI__builtin_ia32_aesencwide128kl_u8:
15159 IID = Intrinsic::x86_aesencwide128kl;
15160 BlockName =
"aesencwide128kl";
15162 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
15163 IID = Intrinsic::x86_aesdecwide128kl;
15164 BlockName =
"aesdecwide128kl";
15166 case X86::BI__builtin_ia32_aesencwide256kl_u8:
15167 IID = Intrinsic::x86_aesencwide256kl;
15168 BlockName =
"aesencwide256kl";
15170 case X86::BI__builtin_ia32_aesdecwide256kl_u8:
15171 IID = Intrinsic::x86_aesdecwide256kl;
15172 BlockName =
"aesdecwide256kl";
15176 llvm::Type *Ty = FixedVectorType::get(Builder.getInt64Ty(), 2);
15179 for (
int i = 0; i != 8; ++i) {
15180 Value *Ptr = Builder.CreateConstGEP1_32(Ty, Ops[1], i);
15181 InOps[i + 1] = Builder.CreateAlignedLoad(Ty, Ptr, Align(16));
15186 BasicBlock *NoError =
15191 Value *
Ret = Builder.CreateExtractValue(Call, 0);
15192 Value *Succ = Builder.CreateTrunc(
Ret, Builder.getInt1Ty());
15193 Builder.CreateCondBr(Succ, NoError,
Error);
15195 Builder.SetInsertPoint(NoError);
15196 for (
int i = 0; i != 8; ++i) {
15197 Value *Extract = Builder.CreateExtractValue(Call, i + 1);
15198 Value *Ptr = Builder.CreateConstGEP1_32(Extract->getType(), Ops[0], i);
15199 Builder.CreateAlignedStore(Extract, Ptr, Align(16));
15201 Builder.CreateBr(End);
15203 Builder.SetInsertPoint(
Error);
15204 for (
int i = 0; i != 8; ++i) {
15205 Value *Out = Builder.CreateExtractValue(Call, i + 1);
15206 Constant *
Zero = llvm::Constant::getNullValue(Out->getType());
15207 Value *Ptr = Builder.CreateConstGEP1_32(Out->getType(), Ops[0], i);
15208 Builder.CreateAlignedStore(Zero, Ptr, Align(16));
15210 Builder.CreateBr(End);
15212 Builder.SetInsertPoint(End);
15213 return Builder.CreateExtractValue(Call, 0);
15215 case X86::BI__builtin_ia32_vfcmaddcph512_mask:
15218 case X86::BI__builtin_ia32_vfmaddcph512_mask: {
15219 Intrinsic::ID IID = IsConjFMA
15220 ? Intrinsic::x86_avx512fp16_mask_vfcmadd_cph_512
15221 : Intrinsic::x86_avx512fp16_mask_vfmadd_cph_512;
15225 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask:
15228 case X86::BI__builtin_ia32_vfmaddcsh_round_mask: {
15229 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
15230 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
15232 Value *
And = Builder.CreateAnd(Ops[3], llvm::ConstantInt::get(
Int8Ty, 1));
15235 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask3:
15238 case X86::BI__builtin_ia32_vfmaddcsh_round_mask3: {
15239 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
15240 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
15242 static constexpr int Mask[] = {0, 5, 6, 7};
15243 return Builder.CreateShuffleVector(Call, Ops[2], Mask);
15262 Intrinsic::ID ID = Intrinsic::not_intrinsic;
15264 switch (BuiltinID) {
15265 default:
return nullptr;
15269 case PPC::BI__builtin_ppc_get_timebase:
15270 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::readcyclecounter));
15273 case PPC::BI__builtin_altivec_lvx:
15274 case PPC::BI__builtin_altivec_lvxl:
15275 case PPC::BI__builtin_altivec_lvebx:
15276 case PPC::BI__builtin_altivec_lvehx:
15277 case PPC::BI__builtin_altivec_lvewx:
15278 case PPC::BI__builtin_altivec_lvsl:
15279 case PPC::BI__builtin_altivec_lvsr:
15280 case PPC::BI__builtin_vsx_lxvd2x:
15281 case PPC::BI__builtin_vsx_lxvw4x:
15282 case PPC::BI__builtin_vsx_lxvd2x_be:
15283 case PPC::BI__builtin_vsx_lxvw4x_be:
15284 case PPC::BI__builtin_vsx_lxvl:
15285 case PPC::BI__builtin_vsx_lxvll:
15290 if(BuiltinID == PPC::BI__builtin_vsx_lxvl ||
15291 BuiltinID == PPC::BI__builtin_vsx_lxvll){
15292 Ops[0] = Builder.CreateBitCast(Ops[0],
Int8PtrTy);
15294 Ops[1] = Builder.CreateBitCast(Ops[1],
Int8PtrTy);
15295 Ops[0] = Builder.CreateGEP(
Int8Ty, Ops[1], Ops[0]);
15299 switch (BuiltinID) {
15300 default: llvm_unreachable(
"Unsupported ld/lvsl/lvsr intrinsic!");
15301 case PPC::BI__builtin_altivec_lvx:
15302 ID = Intrinsic::ppc_altivec_lvx;
15304 case PPC::BI__builtin_altivec_lvxl:
15305 ID = Intrinsic::ppc_altivec_lvxl;
15307 case PPC::BI__builtin_altivec_lvebx:
15308 ID = Intrinsic::ppc_altivec_lvebx;
15310 case PPC::BI__builtin_altivec_lvehx:
15311 ID = Intrinsic::ppc_altivec_lvehx;
15313 case PPC::BI__builtin_altivec_lvewx:
15314 ID = Intrinsic::ppc_altivec_lvewx;
15316 case PPC::BI__builtin_altivec_lvsl:
15317 ID = Intrinsic::ppc_altivec_lvsl;
15319 case PPC::BI__builtin_altivec_lvsr:
15320 ID = Intrinsic::ppc_altivec_lvsr;
15322 case PPC::BI__builtin_vsx_lxvd2x:
15323 ID = Intrinsic::ppc_vsx_lxvd2x;
15325 case PPC::BI__builtin_vsx_lxvw4x:
15326 ID = Intrinsic::ppc_vsx_lxvw4x;
15328 case PPC::BI__builtin_vsx_lxvd2x_be:
15329 ID = Intrinsic::ppc_vsx_lxvd2x_be;
15331 case PPC::BI__builtin_vsx_lxvw4x_be:
15332 ID = Intrinsic::ppc_vsx_lxvw4x_be;
15334 case PPC::BI__builtin_vsx_lxvl:
15335 ID = Intrinsic::ppc_vsx_lxvl;
15337 case PPC::BI__builtin_vsx_lxvll:
15338 ID = Intrinsic::ppc_vsx_lxvll;
15342 return Builder.CreateCall(F, Ops,
"");
15346 case PPC::BI__builtin_altivec_stvx:
15347 case PPC::BI__builtin_altivec_stvxl:
15348 case PPC::BI__builtin_altivec_stvebx:
15349 case PPC::BI__builtin_altivec_stvehx:
15350 case PPC::BI__builtin_altivec_stvewx:
15351 case PPC::BI__builtin_vsx_stxvd2x:
15352 case PPC::BI__builtin_vsx_stxvw4x:
15353 case PPC::BI__builtin_vsx_stxvd2x_be:
15354 case PPC::BI__builtin_vsx_stxvw4x_be:
15355 case PPC::BI__builtin_vsx_stxvl:
15356 case PPC::BI__builtin_vsx_stxvll:
15362 if(BuiltinID == PPC::BI__builtin_vsx_stxvl ||
15363 BuiltinID == PPC::BI__builtin_vsx_stxvll ){
15364 Ops[1] = Builder.CreateBitCast(Ops[1],
Int8PtrTy);
15366 Ops[2] = Builder.CreateBitCast(Ops[2],
Int8PtrTy);
15367 Ops[1] = Builder.CreateGEP(
Int8Ty, Ops[2], Ops[1]);
15371 switch (BuiltinID) {
15372 default: llvm_unreachable(
"Unsupported st intrinsic!");
15373 case PPC::BI__builtin_altivec_stvx:
15374 ID = Intrinsic::ppc_altivec_stvx;
15376 case PPC::BI__builtin_altivec_stvxl:
15377 ID = Intrinsic::ppc_altivec_stvxl;
15379 case PPC::BI__builtin_altivec_stvebx:
15380 ID = Intrinsic::ppc_altivec_stvebx;
15382 case PPC::BI__builtin_altivec_stvehx:
15383 ID = Intrinsic::ppc_altivec_stvehx;
15385 case PPC::BI__builtin_altivec_stvewx:
15386 ID = Intrinsic::ppc_altivec_stvewx;
15388 case PPC::BI__builtin_vsx_stxvd2x:
15389 ID = Intrinsic::ppc_vsx_stxvd2x;
15391 case PPC::BI__builtin_vsx_stxvw4x:
15392 ID = Intrinsic::ppc_vsx_stxvw4x;
15394 case PPC::BI__builtin_vsx_stxvd2x_be:
15395 ID = Intrinsic::ppc_vsx_stxvd2x_be;
15397 case PPC::BI__builtin_vsx_stxvw4x_be:
15398 ID = Intrinsic::ppc_vsx_stxvw4x_be;
15400 case PPC::BI__builtin_vsx_stxvl:
15401 ID = Intrinsic::ppc_vsx_stxvl;
15403 case PPC::BI__builtin_vsx_stxvll:
15404 ID = Intrinsic::ppc_vsx_stxvll;
15408 return Builder.CreateCall(F, Ops,
"");
15410 case PPC::BI__builtin_vsx_ldrmb: {
15421 if (NumBytes == 16) {
15422 Value *BC = Builder.CreateBitCast(Op0, ResTy->getPointerTo());
15430 for (
int Idx = 0; Idx < 16; Idx++)
15431 RevMask.push_back(15 - Idx);
15432 return Builder.CreateShuffleVector(LD, LD, RevMask);
15436 llvm::Function *Lvs =
CGM.
getIntrinsic(IsLE ? Intrinsic::ppc_altivec_lvsr
15437 : Intrinsic::ppc_altivec_lvsl);
15438 llvm::Function *Vperm =
CGM.
getIntrinsic(Intrinsic::ppc_altivec_vperm);
15439 Value *HiMem = Builder.CreateGEP(
15440 Int8Ty, Op0, ConstantInt::get(Op1->getType(), NumBytes - 1));
15441 Value *LoLd = Builder.CreateCall(Lvx, Op0,
"ld.lo");
15442 Value *HiLd = Builder.CreateCall(Lvx, HiMem,
"ld.hi");
15443 Value *Mask1 = Builder.CreateCall(Lvs, Op0,
"mask1");
15445 Op0 = IsLE ? HiLd : LoLd;
15446 Op1 = IsLE ? LoLd : HiLd;
15447 Value *AllElts = Builder.CreateCall(Vperm, {Op0, Op1, Mask1},
"shuffle1");
15448 Constant *
Zero = llvm::Constant::getNullValue(IsLE ? ResTy : AllElts->getType());
15452 for (
int Idx = 0; Idx < 16; Idx++) {
15453 int Val = (NumBytes - Idx - 1 >= 0) ? (NumBytes - Idx - 1)
15454 : 16 - (NumBytes - Idx);
15455 Consts.push_back(Val);
15457 return Builder.CreateShuffleVector(Builder.CreateBitCast(AllElts, ResTy),
15461 for (
int Idx = 0; Idx < 16; Idx++)
15462 Consts.push_back(Builder.getInt8(NumBytes + Idx));
15463 Value *Mask2 = ConstantVector::get(Consts);
15464 return Builder.CreateBitCast(
15465 Builder.CreateCall(Vperm, {Zero, AllElts, Mask2},
"shuffle2"), ResTy);
15467 case PPC::BI__builtin_vsx_strmb: {
15473 auto StoreSubVec = [&](
unsigned Width,
unsigned Offset,
unsigned EltNo) {
15477 Value *BC = Builder.CreateBitCast(Op0, Op2->getType()->getPointerTo());
15478 Value *StVec = Op2;
15481 for (
int Idx = 0; Idx < 16; Idx++)
15482 RevMask.push_back(15 - Idx);
15483 StVec = Builder.CreateShuffleVector(Op2, Op2, RevMask);
15485 return Builder.CreateStore(
15489 unsigned NumElts = 0;
15492 llvm_unreachable(
"width for stores must be a power of 2");
15510 Value *Vec = Builder.CreateBitCast(
15511 Op2, llvm::FixedVectorType::get(ConvTy, NumElts));
15514 Value *PtrBC = Builder.CreateBitCast(Ptr, ConvTy->getPointerTo());
15515 Value *Elt = Builder.CreateExtractElement(Vec, EltNo);
15516 if (IsLE && Width > 1) {
15518 Elt = Builder.CreateCall(F, Elt);
15520 return Builder.CreateStore(
15523 unsigned Stored = 0;
15524 unsigned RemainingBytes = NumBytes;
15526 if (NumBytes == 16)
15527 return StoreSubVec(16, 0, 0);
15528 if (NumBytes >= 8) {
15529 Result = StoreSubVec(8, NumBytes - 8, IsLE ? 0 : 1);
15530 RemainingBytes -= 8;
15533 if (RemainingBytes >= 4) {
15534 Result = StoreSubVec(4, NumBytes - Stored - 4,
15535 IsLE ? (Stored >> 2) : 3 - (Stored >> 2));
15536 RemainingBytes -= 4;
15539 if (RemainingBytes >= 2) {
15540 Result = StoreSubVec(2, NumBytes - Stored - 2,
15541 IsLE ? (Stored >> 1) : 7 - (Stored >> 1));
15542 RemainingBytes -= 2;
15545 if (RemainingBytes)
15547 StoreSubVec(1, NumBytes - Stored - 1, IsLE ? Stored : 15 - Stored);
15551 case PPC::BI__builtin_vsx_xvsqrtsp:
15552 case PPC::BI__builtin_vsx_xvsqrtdp: {
15555 if (Builder.getIsFPConstrained()) {
15557 Intrinsic::experimental_constrained_sqrt, ResultType);
15558 return Builder.CreateConstrainedFPCall(F,
X);
15561 return Builder.CreateCall(F,
X);
15565 case PPC::BI__builtin_altivec_vclzb:
15566 case PPC::BI__builtin_altivec_vclzh:
15567 case PPC::BI__builtin_altivec_vclzw:
15568 case PPC::BI__builtin_altivec_vclzd: {
15571 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
15573 return Builder.CreateCall(F, {
X, Undef});
15575 case PPC::BI__builtin_altivec_vctzb:
15576 case PPC::BI__builtin_altivec_vctzh:
15577 case PPC::BI__builtin_altivec_vctzw:
15578 case PPC::BI__builtin_altivec_vctzd: {
15581 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
15583 return Builder.CreateCall(F, {
X, Undef});
15585 case PPC::BI__builtin_altivec_vinsd:
15586 case PPC::BI__builtin_altivec_vinsw:
15587 case PPC::BI__builtin_altivec_vinsd_elt:
15588 case PPC::BI__builtin_altivec_vinsw_elt: {
15594 bool IsUnaligned = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
15595 BuiltinID == PPC::BI__builtin_altivec_vinsd);
15597 bool Is32bit = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
15598 BuiltinID == PPC::BI__builtin_altivec_vinsw_elt);
15601 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
15603 "Third Arg to vinsw/vinsd intrinsic must be a constant integer!");
15607 int ValidMaxValue = 0;
15609 ValidMaxValue = (Is32bit) ? 12 : 8;
15611 ValidMaxValue = (Is32bit) ? 3 : 1;
15614 int64_t ConstArg = ArgCI->getSExtValue();
15617 std::string RangeErrMsg = IsUnaligned ?
"byte" :
"element";
15618 RangeErrMsg +=
" number " + llvm::to_string(ConstArg);
15619 RangeErrMsg +=
" is outside of the valid range [0, ";
15620 RangeErrMsg += llvm::to_string(ValidMaxValue) +
"]";
15623 if (ConstArg < 0 || ConstArg > ValidMaxValue)
15627 if (!IsUnaligned) {
15628 ConstArg *= Is32bit ? 4 : 8;
15631 ConstArg = (Is32bit ? 12 : 8) - ConstArg;
15634 ID = Is32bit ? Intrinsic::ppc_altivec_vinsw : Intrinsic::ppc_altivec_vinsd;
15635 Op2 = ConstantInt::getSigned(
Int32Ty, ConstArg);
15639 ? Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4))
15640 : Builder.CreateBitCast(Op0,
15641 llvm::FixedVectorType::get(
Int64Ty, 2));
15642 return Builder.CreateBitCast(
15643 Builder.CreateCall(
CGM.
getIntrinsic(ID), {Op0, Op1, Op2}), ResultType);
15645 case PPC::BI__builtin_altivec_vpopcntb:
15646 case PPC::BI__builtin_altivec_vpopcnth:
15647 case PPC::BI__builtin_altivec_vpopcntw:
15648 case PPC::BI__builtin_altivec_vpopcntd: {
15652 return Builder.CreateCall(F,
X);
15654 case PPC::BI__builtin_altivec_vadduqm:
15655 case PPC::BI__builtin_altivec_vsubuqm: {
15658 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
15659 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int128Ty, 1));
15660 Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int128Ty, 1));
15661 if (BuiltinID == PPC::BI__builtin_altivec_vadduqm)
15662 return Builder.CreateAdd(Op0, Op1,
"vadduqm");
15664 return Builder.CreateSub(Op0, Op1,
"vsubuqm");
15666 case PPC::BI__builtin_altivec_vaddcuq_c:
15667 case PPC::BI__builtin_altivec_vsubcuq_c: {
15671 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
15673 Ops.push_back(Builder.CreateBitCast(Op0, V1I128Ty));
15674 Ops.push_back(Builder.CreateBitCast(Op1, V1I128Ty));
15675 ID = (BuiltinID == PPC::BI__builtin_altivec_vaddcuq_c)
15676 ? Intrinsic::ppc_altivec_vaddcuq
15677 : Intrinsic::ppc_altivec_vsubcuq;
15680 case PPC::BI__builtin_altivec_vaddeuqm_c:
15681 case PPC::BI__builtin_altivec_vaddecuq_c:
15682 case PPC::BI__builtin_altivec_vsubeuqm_c:
15683 case PPC::BI__builtin_altivec_vsubecuq_c: {
15688 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
15690 Ops.push_back(Builder.CreateBitCast(Op0, V1I128Ty));
15691 Ops.push_back(Builder.CreateBitCast(Op1, V1I128Ty));
15692 Ops.push_back(Builder.CreateBitCast(Op2, V1I128Ty));
15693 switch (BuiltinID) {
15695 llvm_unreachable(
"Unsupported intrinsic!");
15696 case PPC::BI__builtin_altivec_vaddeuqm_c:
15697 ID = Intrinsic::ppc_altivec_vaddeuqm;
15699 case PPC::BI__builtin_altivec_vaddecuq_c:
15700 ID = Intrinsic::ppc_altivec_vaddecuq;
15702 case PPC::BI__builtin_altivec_vsubeuqm_c:
15703 ID = Intrinsic::ppc_altivec_vsubeuqm;
15705 case PPC::BI__builtin_altivec_vsubecuq_c:
15706 ID = Intrinsic::ppc_altivec_vsubecuq;
15716 case PPC::BI__builtin_ppc_rldimi:
15717 case PPC::BI__builtin_ppc_rlwimi: {
15722 llvm::Type *Ty = Op0->getType();
15724 if (BuiltinID == PPC::BI__builtin_ppc_rldimi)
15725 Op2 = Builder.CreateZExt(Op2,
Int64Ty);
15726 Value *
Shift = Builder.CreateCall(F, {Op0, Op0, Op2});
15727 Value *
X = Builder.CreateAnd(Shift, Op3);
15728 Value *Y = Builder.CreateAnd(Op1, Builder.CreateNot(Op3));
15729 return Builder.CreateOr(
X, Y);
15734 case PPC::BI__builtin_ppc_rlwnm: {
15738 llvm::Type *Ty = Op0->getType();
15740 Value *
Shift = Builder.CreateCall(F, {Op0, Op0, Op1});
15741 return Builder.CreateAnd(Shift, Op2);
15743 case PPC::BI__builtin_ppc_poppar4:
15744 case PPC::BI__builtin_ppc_poppar8: {
15746 llvm::Type *ArgType = Op0->getType();
15748 Value *Tmp = Builder.CreateCall(F, Op0);
15751 Value *
Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
15752 if (
Result->getType() != ResultType)
15753 Result = Builder.CreateIntCast(
Result, ResultType,
true,
15757 case PPC::BI__builtin_ppc_cmpb: {
15760 if (
getTarget().getTriple().isPPC64()) {
15763 return Builder.CreateCall(F, {Op0, Op1},
"cmpb");
15783 Constant *ShiftAmt = ConstantInt::get(
Int64Ty, 32);
15785 Builder.CreateTrunc(Builder.CreateLShr(Op0, ShiftAmt),
Int32Ty);
15787 Builder.CreateTrunc(Builder.CreateLShr(Op1, ShiftAmt),
Int32Ty);
15788 Value *ResLo = Builder.CreateZExt(
15789 Builder.CreateCall(F, {ArgOneLo, ArgTwoLo},
"cmpb"),
Int64Ty);
15790 Value *ResHiShift = Builder.CreateZExt(
15791 Builder.CreateCall(F, {ArgOneHi, ArgTwoHi},
"cmpb"),
Int64Ty);
15792 Value *ResHi = Builder.CreateShl(ResHiShift, ShiftAmt);
15793 return Builder.CreateOr(ResLo, ResHi);
15796 case PPC::BI__builtin_vsx_xvcpsgnsp:
15797 case PPC::BI__builtin_vsx_xvcpsgndp: {
15801 ID = Intrinsic::copysign;
15803 return Builder.CreateCall(F, {
X, Y});
15806 case PPC::BI__builtin_vsx_xvrspip:
15807 case PPC::BI__builtin_vsx_xvrdpip:
15808 case PPC::BI__builtin_vsx_xvrdpim:
15809 case PPC::BI__builtin_vsx_xvrspim:
15810 case PPC::BI__builtin_vsx_xvrdpi:
15811 case PPC::BI__builtin_vsx_xvrspi:
15812 case PPC::BI__builtin_vsx_xvrdpic:
15813 case PPC::BI__builtin_vsx_xvrspic:
15814 case PPC::BI__builtin_vsx_xvrdpiz:
15815 case PPC::BI__builtin_vsx_xvrspiz: {
15818 if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim ||
15819 BuiltinID == PPC::BI__builtin_vsx_xvrspim)
15820 ID = Builder.getIsFPConstrained()
15821 ? Intrinsic::experimental_constrained_floor
15822 : Intrinsic::floor;
15823 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi ||
15824 BuiltinID == PPC::BI__builtin_vsx_xvrspi)
15825 ID = Builder.getIsFPConstrained()
15826 ? Intrinsic::experimental_constrained_round
15827 : Intrinsic::round;
15828 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic ||
15829 BuiltinID == PPC::BI__builtin_vsx_xvrspic)
15830 ID = Builder.getIsFPConstrained()
15831 ? Intrinsic::experimental_constrained_rint
15833 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip ||
15834 BuiltinID == PPC::BI__builtin_vsx_xvrspip)
15835 ID = Builder.getIsFPConstrained()
15836 ? Intrinsic::experimental_constrained_ceil
15838 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz ||
15839 BuiltinID == PPC::BI__builtin_vsx_xvrspiz)
15840 ID = Builder.getIsFPConstrained()
15841 ? Intrinsic::experimental_constrained_trunc
15842 : Intrinsic::trunc;
15844 return Builder.getIsFPConstrained() ? Builder.CreateConstrainedFPCall(F,
X)
15845 : Builder.CreateCall(F,
X);
15849 case PPC::BI__builtin_vsx_xvabsdp:
15850 case PPC::BI__builtin_vsx_xvabssp: {
15854 return Builder.CreateCall(F,
X);
15858 case PPC::BI__builtin_ppc_recipdivf:
15859 case PPC::BI__builtin_ppc_recipdivd:
15860 case PPC::BI__builtin_ppc_rsqrtf:
15861 case PPC::BI__builtin_ppc_rsqrtd: {
15862 FastMathFlags FMF = Builder.getFastMathFlags();
15863 Builder.getFastMathFlags().setFast();
15867 if (BuiltinID == PPC::BI__builtin_ppc_recipdivf ||
15868 BuiltinID == PPC::BI__builtin_ppc_recipdivd) {
15870 Value *FDiv = Builder.CreateFDiv(
X, Y,
"recipdiv");
15871 Builder.getFastMathFlags() &= (FMF);
15874 auto *One = ConstantFP::get(ResultType, 1.0);
15876 Value *FDiv = Builder.CreateFDiv(One, Builder.CreateCall(F,
X),
"rsqrt");
15877 Builder.getFastMathFlags() &= (FMF);
15880 case PPC::BI__builtin_ppc_alignx: {
15884 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
15885 AlignmentCI = ConstantInt::get(AlignmentCI->getType(),
15886 llvm::Value::MaximumAlignment);
15890 AlignmentCI,
nullptr);
15893 case PPC::BI__builtin_ppc_rdlam: {
15897 llvm::Type *Ty = Op0->getType();
15898 Value *ShiftAmt = Builder.CreateIntCast(Op1, Ty,
false);
15900 Value *Rotate = Builder.CreateCall(F, {Op0, Op0, ShiftAmt});
15901 return Builder.CreateAnd(Rotate, Op2);
15903 case PPC::BI__builtin_ppc_load2r: {
15906 Value *LoadIntrinsic = Builder.CreateCall(F, {Op0});
15907 return Builder.CreateTrunc(LoadIntrinsic,
Int16Ty);
15910 case PPC::BI__builtin_ppc_fnmsub:
15911 case PPC::BI__builtin_ppc_fnmsubs:
15912 case PPC::BI__builtin_vsx_xvmaddadp:
15913 case PPC::BI__builtin_vsx_xvmaddasp:
15914 case PPC::BI__builtin_vsx_xvnmaddadp:
15915 case PPC::BI__builtin_vsx_xvnmaddasp:
15916 case PPC::BI__builtin_vsx_xvmsubadp:
15917 case PPC::BI__builtin_vsx_xvmsubasp:
15918 case PPC::BI__builtin_vsx_xvnmsubadp:
15919 case PPC::BI__builtin_vsx_xvnmsubasp: {
15925 if (Builder.getIsFPConstrained())
15926 F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
15929 switch (BuiltinID) {
15930 case PPC::BI__builtin_vsx_xvmaddadp:
15931 case PPC::BI__builtin_vsx_xvmaddasp:
15932 if (Builder.getIsFPConstrained())
15933 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
15935 return Builder.CreateCall(F, {
X, Y, Z});
15936 case PPC::BI__builtin_vsx_xvnmaddadp:
15937 case PPC::BI__builtin_vsx_xvnmaddasp:
15938 if (Builder.getIsFPConstrained())
15939 return Builder.CreateFNeg(
15940 Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
15942 return Builder.CreateFNeg(Builder.CreateCall(F, {X, Y, Z}),
"neg");
15943 case PPC::BI__builtin_vsx_xvmsubadp:
15944 case PPC::BI__builtin_vsx_xvmsubasp:
15945 if (Builder.getIsFPConstrained())
15946 return Builder.CreateConstrainedFPCall(
15947 F, {
X, Y, Builder.CreateFNeg(Z,
"neg")});
15949 return Builder.CreateCall(F, {
X, Y, Builder.CreateFNeg(Z,
"neg")});
15950 case PPC::BI__builtin_ppc_fnmsub:
15951 case PPC::BI__builtin_ppc_fnmsubs:
15952 case PPC::BI__builtin_vsx_xvnmsubadp:
15953 case PPC::BI__builtin_vsx_xvnmsubasp:
15954 if (Builder.getIsFPConstrained())
15955 return Builder.CreateFNeg(
15956 Builder.CreateConstrainedFPCall(
15957 F, {X, Y, Builder.CreateFNeg(Z,
"neg")}),
15960 return Builder.CreateCall(
15963 llvm_unreachable(
"Unknown FMA operation");
15967 case PPC::BI__builtin_vsx_insertword: {
15975 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
15977 "Third arg to xxinsertw intrinsic must be constant integer");
15979 int64_t Index =
clamp(ArgCI->getSExtValue(), 0, MaxIndex);
15986 std::swap(Op0, Op1);
15990 Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int64Ty, 2));
15994 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
15995 Op0 = Builder.CreateShuffleVector(Op0, Op0,
ArrayRef<int>{1, 0});
15998 Index = MaxIndex - Index;
16002 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4));
16003 Op2 = ConstantInt::getSigned(
Int32Ty, Index);
16004 return Builder.CreateCall(F, {Op0, Op1, Op2});
16007 case PPC::BI__builtin_vsx_extractuword: {
16010 llvm::Function *F =
CGM.
getIntrinsic(Intrinsic::ppc_vsx_xxextractuw);
16013 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
16017 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op1);
16019 "Second Arg to xxextractuw intrinsic must be a constant integer!");
16021 int64_t Index =
clamp(ArgCI->getSExtValue(), 0, MaxIndex);
16025 Index = MaxIndex - Index;
16026 Op1 = ConstantInt::getSigned(
Int32Ty, Index);
16029 Value *
Call = Builder.CreateCall(F, {Op0, Op1});
16031 Value *ShuffleCall =
16032 Builder.CreateShuffleVector(Call, Call,
ArrayRef<int>{1, 0});
16033 return ShuffleCall;
16035 Op1 = ConstantInt::getSigned(
Int32Ty, Index);
16036 return Builder.CreateCall(F, {Op0, Op1});
16040 case PPC::BI__builtin_vsx_xxpermdi: {
16044 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
16045 assert(ArgCI &&
"Third arg must be constant integer!");
16047 unsigned Index = ArgCI->getZExtValue();
16048 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
16049 Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int64Ty, 2));
16054 int ElemIdx0 = (Index & 2) >> 1;
16055 int ElemIdx1 = 2 + (Index & 1);
16057 int ShuffleElts[2] = {ElemIdx0, ElemIdx1};
16058 Value *ShuffleCall = Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
16061 return Builder.CreateBitCast(ShuffleCall, RetTy);
16064 case PPC::BI__builtin_vsx_xxsldwi: {
16068 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
16069 assert(ArgCI &&
"Third argument must be a compile time constant");
16070 unsigned Index = ArgCI->getZExtValue() & 0x3;
16071 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4));
16072 Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int32Ty, 4));
16083 ElemIdx0 = (8 - Index) % 8;
16084 ElemIdx1 = (9 - Index) % 8;
16085 ElemIdx2 = (10 - Index) % 8;
16086 ElemIdx3 = (11 - Index) % 8;
16090 ElemIdx1 = Index + 1;
16091 ElemIdx2 = Index + 2;
16092 ElemIdx3 = Index + 3;
16095 int ShuffleElts[4] = {ElemIdx0, ElemIdx1, ElemIdx2, ElemIdx3};
16096 Value *ShuffleCall = Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
16099 return Builder.CreateBitCast(ShuffleCall, RetTy);
16102 case PPC::BI__builtin_pack_vector_int128: {
16106 Value *UndefValue =
16107 llvm::UndefValue::get(llvm::FixedVectorType::get(Op0->getType(), 2));
16108 Value *Res = Builder.CreateInsertElement(
16109 UndefValue, Op0, (
uint64_t)(isLittleEndian ? 1 : 0));
16110 Res = Builder.CreateInsertElement(Res, Op1,
16111 (
uint64_t)(isLittleEndian ? 0 : 1));
16115 case PPC::BI__builtin_unpack_vector_int128: {
16119 Value *Unpacked = Builder.CreateBitCast(
16123 Index = ConstantInt::get(Index->getType(), 1 - Index->getZExtValue());
16125 return Builder.CreateExtractElement(Unpacked, Index);
16128 case PPC::BI__builtin_ppc_sthcx: {
16132 return Builder.CreateCall(F, {Op0, Op1});
16141#define CUSTOM_BUILTIN(Name, Intr, Types, Accumulate) \
16142 case PPC::BI__builtin_##Name:
16143#include "clang/Basic/BuiltinsPPC.def"
16146 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++)
16155 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc ||
16156 BuiltinID == PPC::BI__builtin_vsx_disassemble_pair ||
16157 BuiltinID == PPC::BI__builtin_mma_disassemble_pair) {
16158 unsigned NumVecs = 2;
16159 auto Intrinsic = Intrinsic::ppc_vsx_disassemble_pair;
16160 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc) {
16162 Intrinsic = Intrinsic::ppc_mma_disassemble_acc;
16166 Value *Vec = Builder.CreateLoad(Addr);
16167 Value *
Call = Builder.CreateCall(F, {Vec});
16168 llvm::Type *VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
16169 Value *Ptr = Builder.CreateBitCast(Ops[0], VTy->getPointerTo());
16170 for (
unsigned i=0; i<NumVecs; i++) {
16171 Value *Vec = Builder.CreateExtractValue(Call, i);
16172 llvm::ConstantInt* Index = llvm::ConstantInt::get(
IntTy, i);
16173 Value *GEP = Builder.CreateInBoundsGEP(VTy, Ptr, Index);
16174 Builder.CreateAlignedStore(Vec, GEP, MaybeAlign(16));
16178 if (BuiltinID == PPC::BI__builtin_vsx_build_pair ||
16179 BuiltinID == PPC::BI__builtin_mma_build_acc) {
16187 std::reverse(Ops.begin() + 1, Ops.end());
16190 switch (BuiltinID) {
16191 #define CUSTOM_BUILTIN(Name, Intr, Types, Acc) \
16192 case PPC::BI__builtin_##Name: \
16193 ID = Intrinsic::ppc_##Intr; \
16194 Accumulate = Acc; \
16196 #include "clang/Basic/BuiltinsPPC.def"
16198 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
16199 BuiltinID == PPC::BI__builtin_vsx_stxvp ||
16200 BuiltinID == PPC::BI__builtin_mma_lxvp ||
16201 BuiltinID == PPC::BI__builtin_mma_stxvp) {
16202 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
16203 BuiltinID == PPC::BI__builtin_mma_lxvp) {
16204 Ops[1] = Builder.CreateBitCast(Ops[1],
Int8PtrTy);
16205 Ops[0] = Builder.CreateGEP(
Int8Ty, Ops[1], Ops[0]);
16207 Ops[2] = Builder.CreateBitCast(Ops[2],
Int8PtrTy);
16208 Ops[1] = Builder.CreateGEP(
Int8Ty, Ops[2], Ops[1]);
16212 return Builder.CreateCall(F, Ops,
"");
16217 Value *Acc = Builder.CreateLoad(Addr);
16218 CallOps.push_back(Acc);
16220 for (
unsigned i=1; i<Ops.size(); i++)
16221 CallOps.push_back(Ops[i]);
16223 Value *
Call = Builder.CreateCall(F, CallOps);
16224 return Builder.CreateAlignedStore(Call, Ops[0], MaybeAlign(64));
16227 case PPC::BI__builtin_ppc_compare_and_swap:
16228 case PPC::BI__builtin_ppc_compare_and_swaplp: {
16231 Value *OldVal = Builder.CreateLoad(OldValAddr);
16237 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Monotonic,
true);
16245 Value *LoadedVal = Pair.first.getScalarVal();
16246 Builder.CreateStore(LoadedVal, OldValAddr);
16247 return Builder.CreateZExt(Pair.second, Builder.getInt32Ty());
16249 case PPC::BI__builtin_ppc_fetch_and_add:
16250 case PPC::BI__builtin_ppc_fetch_and_addlp: {
16252 llvm::AtomicOrdering::Monotonic);
16254 case PPC::BI__builtin_ppc_fetch_and_and:
16255 case PPC::BI__builtin_ppc_fetch_and_andlp: {
16257 llvm::AtomicOrdering::Monotonic);
16260 case PPC::BI__builtin_ppc_fetch_and_or:
16261 case PPC::BI__builtin_ppc_fetch_and_orlp: {
16263 llvm::AtomicOrdering::Monotonic);
16265 case PPC::BI__builtin_ppc_fetch_and_swap:
16266 case PPC::BI__builtin_ppc_fetch_and_swaplp: {
16268 llvm::AtomicOrdering::Monotonic);
16270 case PPC::BI__builtin_ppc_ldarx:
16271 case PPC::BI__builtin_ppc_lwarx:
16272 case PPC::BI__builtin_ppc_lharx:
16273 case PPC::BI__builtin_ppc_lbarx:
16275 case PPC::BI__builtin_ppc_mfspr: {
16281 return Builder.CreateCall(F, {Op0});
16283 case PPC::BI__builtin_ppc_mtspr: {
16290 return Builder.CreateCall(F, {Op0, Op1});
16292 case PPC::BI__builtin_ppc_popcntb: {
16294 llvm::Type *ArgType = ArgValue->getType();
16295 Function *F =
CGM.
getIntrinsic(Intrinsic::ppc_popcntb, {ArgType, ArgType});
16296 return Builder.CreateCall(F, {ArgValue},
"popcntb");
16298 case PPC::BI__builtin_ppc_mtfsf: {
16305 return Builder.CreateCall(F, {Op0,
Cast},
"");
16308 case PPC::BI__builtin_ppc_swdiv_nochk:
16309 case PPC::BI__builtin_ppc_swdivs_nochk: {
16312 FastMathFlags FMF = Builder.getFastMathFlags();
16313 Builder.getFastMathFlags().setFast();
16314 Value *FDiv = Builder.CreateFDiv(Op0, Op1,
"swdiv_nochk");
16315 Builder.getFastMathFlags() &= (FMF);
16318 case PPC::BI__builtin_ppc_fric:
16320 *
this, E, Intrinsic::rint,
16321 Intrinsic::experimental_constrained_rint))
16323 case PPC::BI__builtin_ppc_frim:
16324 case PPC::BI__builtin_ppc_frims:
16326 *
this, E, Intrinsic::floor,
16327 Intrinsic::experimental_constrained_floor))
16329 case PPC::BI__builtin_ppc_frin:
16330 case PPC::BI__builtin_ppc_frins:
16332 *
this, E, Intrinsic::round,
16333 Intrinsic::experimental_constrained_round))
16335 case PPC::BI__builtin_ppc_frip:
16336 case PPC::BI__builtin_ppc_frips:
16338 *
this, E, Intrinsic::ceil,
16339 Intrinsic::experimental_constrained_ceil))
16341 case PPC::BI__builtin_ppc_friz:
16342 case PPC::BI__builtin_ppc_frizs:
16344 *
this, E, Intrinsic::trunc,
16345 Intrinsic::experimental_constrained_trunc))
16347 case PPC::BI__builtin_ppc_fsqrt:
16348 case PPC::BI__builtin_ppc_fsqrts:
16350 *
this, E, Intrinsic::sqrt,
16351 Intrinsic::experimental_constrained_sqrt))
16353 case PPC::BI__builtin_ppc_test_data_class: {
16356 llvm::Type *ArgType = Op0->getType();
16357 unsigned IntrinsicID;
16358 if (ArgType->isDoubleTy())
16359 IntrinsicID = Intrinsic::ppc_test_data_class_d;
16360 else if (ArgType->isFloatTy())
16361 IntrinsicID = Intrinsic::ppc_test_data_class_f;
16363 llvm_unreachable(
"Invalid Argument Type");
16365 "test_data_class");
16367 case PPC::BI__builtin_ppc_maxfe: {
16373 {Op0, Op1, Op2, Op3});
16375 case PPC::BI__builtin_ppc_maxfl: {
16381 {Op0, Op1, Op2, Op3});
16383 case PPC::BI__builtin_ppc_maxfs: {
16389 {Op0, Op1, Op2, Op3});
16391 case PPC::BI__builtin_ppc_minfe: {
16397 {Op0, Op1, Op2, Op3});
16399 case PPC::BI__builtin_ppc_minfl: {
16405 {Op0, Op1, Op2, Op3});
16407 case PPC::BI__builtin_ppc_minfs: {
16413 {Op0, Op1, Op2, Op3});
16415 case PPC::BI__builtin_ppc_swdiv:
16416 case PPC::BI__builtin_ppc_swdivs: {
16419 return Builder.CreateFDiv(Op0, Op1,
"swdiv");
16432 Attribute::getWithDereferenceableBytes(
Call->getContext(), 64));
16433 Call->addRetAttr(Attribute::getWithAlignment(
Call->getContext(), Align(4)));
16438 if (RetTy ==
Call->getType())
16447 Attribute::getWithDereferenceableBytes(
Call->getContext(), 256));
16448 Call->addRetAttr(Attribute::getWithAlignment(
Call->getContext(), Align(8)));
16460 Offset = llvm::ConstantInt::get(CGF.
Int32Ty, 12 + Index * 2);
16461 DP = EmitAMDGPUImplicitArgPtr(CGF);
16464 Offset = llvm::ConstantInt::get(CGF.
Int32Ty, 4 + Index * 2);
16465 DP = EmitAMDGPUDispatchPtr(CGF);
16470 CGF.
Int16Ty->getPointerTo(GEP->getType()->getPointerAddressSpace());
16471 auto *
Cast = CGF.
Builder.CreateBitCast(GEP, DstTy);
16475 llvm::MDNode *RNode = MDHelper.createRange(APInt(16, 1),
16477 LD->setMetadata(llvm::LLVMContext::MD_range, RNode);
16478 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
16485 const unsigned XOffset = 12;
16486 auto *DP = EmitAMDGPUDispatchPtr(CGF);
16488 auto *
Offset = llvm::ConstantInt::get(CGF.
Int32Ty, XOffset + Index * 4);
16491 CGF.
Int32Ty->getPointerTo(GEP->getType()->getPointerAddressSpace());
16492 auto *
Cast = CGF.
Builder.CreateBitCast(GEP, DstTy);
16495 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
16508 llvm::AtomicOrdering &AO,
16509 llvm::SyncScope::ID &SSID) {
16510 if (isa<llvm::ConstantInt>(Order)) {
16514 assert(llvm::isValidAtomicOrderingCABI(ord));
16515 switch (
static_cast<llvm::AtomicOrderingCABI
>(ord)) {
16516 case llvm::AtomicOrderingCABI::acquire:
16517 case llvm::AtomicOrderingCABI::consume:
16518 AO = llvm::AtomicOrdering::Acquire;
16520 case llvm::AtomicOrderingCABI::release:
16521 AO = llvm::AtomicOrdering::Release;
16523 case llvm::AtomicOrderingCABI::acq_rel:
16524 AO = llvm::AtomicOrdering::AcquireRelease;
16526 case llvm::AtomicOrderingCABI::seq_cst:
16527 AO = llvm::AtomicOrdering::SequentiallyConsistent;
16529 case llvm::AtomicOrderingCABI::relaxed:
16530 AO = llvm::AtomicOrdering::Monotonic;
16535 llvm::getConstantStringInfo(
Scope, scp);
16544 llvm::AtomicOrdering AO = llvm::AtomicOrdering::SequentiallyConsistent;
16545 llvm::SyncScope::ID SSID;
16546 switch (BuiltinID) {
16547 case AMDGPU::BI__builtin_amdgcn_div_scale:
16548 case AMDGPU::BI__builtin_amdgcn_div_scalef: {
16561 llvm::Value *Tmp = Builder.CreateCall(Callee, {
X, Y, Z});
16563 llvm::Value *
Result = Builder.CreateExtractValue(Tmp, 0);
16564 llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1);
16568 llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType);
16569 Builder.CreateStore(FlagExt, FlagOutPtr);
16572 case AMDGPU::BI__builtin_amdgcn_div_fmas:
16573 case AMDGPU::BI__builtin_amdgcn_div_fmasf: {
16581 llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3);
16582 return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
16585 case AMDGPU::BI__builtin_amdgcn_ds_swizzle:
16587 case AMDGPU::BI__builtin_amdgcn_mov_dpp8:
16589 case AMDGPU::BI__builtin_amdgcn_mov_dpp:
16590 case AMDGPU::BI__builtin_amdgcn_update_dpp: {
16592 for (
unsigned I = 0; I != E->
getNumArgs(); ++I)
16594 assert(Args.size() == 5 || Args.size() == 6);
16595 if (Args.size() == 5)
16596 Args.insert(Args.begin(), llvm::UndefValue::get(Args[0]->getType()));
16599 return Builder.CreateCall(F, Args);
16601 case AMDGPU::BI__builtin_amdgcn_div_fixup:
16602 case AMDGPU::BI__builtin_amdgcn_div_fixupf:
16603 case AMDGPU::BI__builtin_amdgcn_div_fixuph:
16605 case AMDGPU::BI__builtin_amdgcn_trig_preop:
16606 case AMDGPU::BI__builtin_amdgcn_trig_preopf:
16608 case AMDGPU::BI__builtin_amdgcn_rcp:
16609 case AMDGPU::BI__builtin_amdgcn_rcpf:
16610 case AMDGPU::BI__builtin_amdgcn_rcph:
16612 case AMDGPU::BI__builtin_amdgcn_sqrt:
16613 case AMDGPU::BI__builtin_amdgcn_sqrtf:
16614 case AMDGPU::BI__builtin_amdgcn_sqrth:
16616 case AMDGPU::BI__builtin_amdgcn_rsq:
16617 case AMDGPU::BI__builtin_amdgcn_rsqf:
16618 case AMDGPU::BI__builtin_amdgcn_rsqh:
16620 case AMDGPU::BI__builtin_amdgcn_rsq_clamp:
16621 case AMDGPU::BI__builtin_amdgcn_rsq_clampf:
16623 case AMDGPU::BI__builtin_amdgcn_sinf:
16624 case AMDGPU::BI__builtin_amdgcn_sinh:
16626 case AMDGPU::BI__builtin_amdgcn_cosf:
16627 case AMDGPU::BI__builtin_amdgcn_cosh:
16629 case AMDGPU::BI__builtin_amdgcn_dispatch_ptr:
16630 return EmitAMDGPUDispatchPtr(*
this, E);
16631 case AMDGPU::BI__builtin_amdgcn_log_clampf:
16633 case AMDGPU::BI__builtin_amdgcn_ldexp:
16634 case AMDGPU::BI__builtin_amdgcn_ldexpf:
16635 case AMDGPU::BI__builtin_amdgcn_ldexph:
16637 case AMDGPU::BI__builtin_amdgcn_frexp_mant:
16638 case AMDGPU::BI__builtin_amdgcn_frexp_mantf:
16639 case AMDGPU::BI__builtin_amdgcn_frexp_manth:
16641 case AMDGPU::BI__builtin_amdgcn_frexp_exp:
16642 case AMDGPU::BI__builtin_amdgcn_frexp_expf: {
16645 { Builder.getInt32Ty(), Src0->getType() });
16646 return Builder.CreateCall(F, Src0);
16648 case AMDGPU::BI__builtin_amdgcn_frexp_exph: {
16651 { Builder.getInt16Ty(), Src0->getType() });
16652 return Builder.CreateCall(F, Src0);
16654 case AMDGPU::BI__builtin_amdgcn_fract:
16655 case AMDGPU::BI__builtin_amdgcn_fractf:
16656 case AMDGPU::BI__builtin_amdgcn_fracth:
16658 case AMDGPU::BI__builtin_amdgcn_lerp:
16660 case AMDGPU::BI__builtin_amdgcn_ubfe:
16662 case AMDGPU::BI__builtin_amdgcn_sbfe:
16664 case AMDGPU::BI__builtin_amdgcn_uicmp:
16665 case AMDGPU::BI__builtin_amdgcn_uicmpl:
16666 case AMDGPU::BI__builtin_amdgcn_sicmp:
16667 case AMDGPU::BI__builtin_amdgcn_sicmpl: {
16674 { Builder.getInt64Ty(), Src0->getType() });
16675 return Builder.CreateCall(F, { Src0, Src1, Src2 });
16677 case AMDGPU::BI__builtin_amdgcn_fcmp:
16678 case AMDGPU::BI__builtin_amdgcn_fcmpf: {
16685 { Builder.getInt64Ty(), Src0->getType() });
16686 return Builder.CreateCall(F, { Src0, Src1, Src2 });
16688 case AMDGPU::BI__builtin_amdgcn_class:
16689 case AMDGPU::BI__builtin_amdgcn_classf:
16690 case AMDGPU::BI__builtin_amdgcn_classh:
16692 case AMDGPU::BI__builtin_amdgcn_fmed3f:
16693 case AMDGPU::BI__builtin_amdgcn_fmed3h:
16695 case AMDGPU::BI__builtin_amdgcn_ds_append:
16696 case AMDGPU::BI__builtin_amdgcn_ds_consume: {
16697 Intrinsic::ID Intrin = BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_append ?
16698 Intrinsic::amdgcn_ds_append : Intrinsic::amdgcn_ds_consume;
16701 return Builder.CreateCall(F, { Src0, Builder.getFalse() });
16703 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
16704 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
16705 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf: {
16706 Intrinsic::ID Intrin;
16707 switch (BuiltinID) {
16708 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
16709 Intrin = Intrinsic::amdgcn_ds_fadd;
16711 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
16712 Intrin = Intrinsic::amdgcn_ds_fmin;
16714 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
16715 Intrin = Intrinsic::amdgcn_ds_fmax;
16724 llvm::FunctionType *FTy = F->getFunctionType();
16725 llvm::Type *PTy = FTy->getParamType(0);
16726 Src0 = Builder.CreatePointerBitCastOrAddrSpaceCast(Src0, PTy);
16727 return Builder.CreateCall(F, { Src0, Src1, Src2, Src3, Src4 });
16729 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
16730 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
16731 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
16732 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
16733 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
16734 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
16735 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
16736 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
16737 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
16738 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16: {
16741 switch (BuiltinID) {
16742 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
16744 IID = Intrinsic::amdgcn_global_atomic_fadd;
16746 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
16747 ArgTy = llvm::FixedVectorType::get(
16749 IID = Intrinsic::amdgcn_global_atomic_fadd;
16751 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
16752 IID = Intrinsic::amdgcn_global_atomic_fadd;
16754 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
16755 IID = Intrinsic::amdgcn_global_atomic_fmin;
16757 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
16758 IID = Intrinsic::amdgcn_global_atomic_fmax;
16760 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
16761 IID = Intrinsic::amdgcn_flat_atomic_fadd;
16763 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
16764 IID = Intrinsic::amdgcn_flat_atomic_fmin;
16766 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
16767 IID = Intrinsic::amdgcn_flat_atomic_fmax;
16769 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
16771 IID = Intrinsic::amdgcn_flat_atomic_fadd;
16773 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16:
16774 ArgTy = llvm::FixedVectorType::get(
16776 IID = Intrinsic::amdgcn_flat_atomic_fadd;
16781 llvm::Function *F =
16783 return Builder.CreateCall(F, {Addr, Val});
16785 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
16786 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16: {
16788 switch (BuiltinID) {
16789 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
16790 IID = Intrinsic::amdgcn_global_atomic_fadd_v2bf16;
16792 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16:
16793 IID = Intrinsic::amdgcn_flat_atomic_fadd_v2bf16;
16799 return Builder.CreateCall(F, {Addr, Val});
16801 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
16802 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32: {
16805 switch (BuiltinID) {
16806 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
16808 IID = Intrinsic::amdgcn_ds_fadd;
16810 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
16812 IID = Intrinsic::amdgcn_ds_fadd;
16817 llvm::Constant *ZeroI32 = llvm::ConstantInt::getIntegerValue(
16819 llvm::Constant *ZeroI1 = llvm::ConstantInt::getIntegerValue(
16822 return Builder.CreateCall(F, {Addr, Val, ZeroI32, ZeroI32, ZeroI1});
16824 case AMDGPU::BI__builtin_amdgcn_read_exec: {
16827 CI->setConvergent();
16830 case AMDGPU::BI__builtin_amdgcn_read_exec_lo:
16831 case AMDGPU::BI__builtin_amdgcn_read_exec_hi: {
16832 StringRef RegName = BuiltinID == AMDGPU::BI__builtin_amdgcn_read_exec_lo ?
16833 "exec_lo" :
"exec_hi";
16836 CI->setConvergent();
16839 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray:
16840 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_h:
16841 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_l:
16842 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_lh: {
16852 RayOrigin = Builder.CreateShuffleVector(RayOrigin, RayOrigin,
16855 Builder.CreateShuffleVector(RayDir, RayDir,
ArrayRef<int>{0, 1, 2});
16856 RayInverseDir = Builder.CreateShuffleVector(RayInverseDir, RayInverseDir,
16859 Function *F =
CGM.
getIntrinsic(Intrinsic::amdgcn_image_bvh_intersect_ray,
16860 {NodePtr->getType(), RayDir->getType()});
16861 return Builder.CreateCall(F, {NodePtr, RayExtent, RayOrigin, RayDir,
16862 RayInverseDir, TextureDescr});
16865 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
16866 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
16867 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
16868 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
16869 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
16870 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
16871 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
16872 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
16873 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
16874 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
16875 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
16876 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64: {
16882 unsigned ArgForMatchingRetType;
16883 unsigned BuiltinWMMAOp;
16885 switch (BuiltinID) {
16886 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
16887 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
16888 ArgForMatchingRetType = 2;
16889 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_f16;
16891 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
16892 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
16893 ArgForMatchingRetType = 2;
16894 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf16;
16896 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
16897 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
16898 ArgForMatchingRetType = 2;
16899 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16;
16901 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
16902 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
16903 ArgForMatchingRetType = 2;
16904 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16;
16906 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
16907 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
16908 ArgForMatchingRetType = 4;
16909 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu8;
16911 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
16912 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
16913 ArgForMatchingRetType = 4;
16914 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu4;
16919 for (
int i = 0, e = E->
getNumArgs(); i != e; ++i)
16923 {Args[ArgForMatchingRetType]->getType()});
16925 return Builder.CreateCall(F, Args);
16929 case AMDGPU::BI__builtin_amdgcn_workitem_id_x:
16931 case AMDGPU::BI__builtin_amdgcn_workitem_id_y:
16933 case AMDGPU::BI__builtin_amdgcn_workitem_id_z:
16937 case AMDGPU::BI__builtin_amdgcn_workgroup_size_x:
16938 return EmitAMDGPUWorkGroupSize(*
this, 0);
16939 case AMDGPU::BI__builtin_amdgcn_workgroup_size_y:
16940 return EmitAMDGPUWorkGroupSize(*
this, 1);
16941 case AMDGPU::BI__builtin_amdgcn_workgroup_size_z:
16942 return EmitAMDGPUWorkGroupSize(*
this, 2);
16945 case AMDGPU::BI__builtin_amdgcn_grid_size_x:
16946 return EmitAMDGPUGridSize(*
this, 0);
16947 case AMDGPU::BI__builtin_amdgcn_grid_size_y:
16948 return EmitAMDGPUGridSize(*
this, 1);
16949 case AMDGPU::BI__builtin_amdgcn_grid_size_z:
16950 return EmitAMDGPUGridSize(*
this, 2);
16953 case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
16954 case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
16956 case AMDGPU::BI__builtin_r600_read_tidig_x:
16958 case AMDGPU::BI__builtin_r600_read_tidig_y:
16960 case AMDGPU::BI__builtin_r600_read_tidig_z:
16962 case AMDGPU::BI__builtin_amdgcn_alignbit: {
16967 return Builder.CreateCall(F, { Src0, Src1, Src2 });
16970 case AMDGPU::BI__builtin_amdgcn_fence: {
16973 return Builder.CreateFence(AO, SSID);
16976 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
16977 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
16978 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
16979 case AMDGPU::BI__builtin_amdgcn_atomic_dec64: {
16980 unsigned BuiltinAtomicOp;
16983 switch (BuiltinID) {
16984 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
16985 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
16986 BuiltinAtomicOp = Intrinsic::amdgcn_atomic_inc;
16988 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
16989 case AMDGPU::BI__builtin_amdgcn_atomic_dec64:
16990 BuiltinAtomicOp = Intrinsic::amdgcn_atomic_dec;
16997 llvm::Function *F =
17005 Value *MemOrder = Builder.getInt32(
static_cast<int>(AO));
17006 Value *MemScope = Builder.getInt32(
static_cast<int>(SSID));
17010 PtrTy->castAs<
PointerType>()->getPointeeType().isVolatileQualified();
17011 Value *IsVolatile = Builder.getInt1(
static_cast<bool>(Volatile));
17013 return Builder.CreateCall(F, {Ptr, Val, MemOrder, MemScope, IsVolatile});
17026 unsigned IntrinsicID,
17030 for (
unsigned I = 0; I < NumArgs; ++I)
17037 return CGF.
Builder.CreateExtractValue(Call, 0);
17042 switch (BuiltinID) {
17043 case SystemZ::BI__builtin_tbegin: {
17045 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
17047 return Builder.CreateCall(F, {TDB, Control});
17049 case SystemZ::BI__builtin_tbegin_nofloat: {
17051 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
17053 return Builder.CreateCall(F, {TDB, Control});
17055 case SystemZ::BI__builtin_tbeginc: {
17057 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff08);
17059 return Builder.CreateCall(F, {TDB, Control});
17061 case SystemZ::BI__builtin_tabort: {
17064 return Builder.CreateCall(F, Builder.CreateSExt(Data,
Int64Ty,
"tabort"));
17066 case SystemZ::BI__builtin_non_tx_store: {
17070 return Builder.CreateCall(F, {Data,
Address});
17078 case SystemZ::BI__builtin_s390_vpopctb:
17079 case SystemZ::BI__builtin_s390_vpopcth:
17080 case SystemZ::BI__builtin_s390_vpopctf:
17081 case SystemZ::BI__builtin_s390_vpopctg: {
17085 return Builder.CreateCall(F,
X);
17088 case SystemZ::BI__builtin_s390_vclzb:
17089 case SystemZ::BI__builtin_s390_vclzh:
17090 case SystemZ::BI__builtin_s390_vclzf:
17091 case SystemZ::BI__builtin_s390_vclzg: {
17094 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
17096 return Builder.CreateCall(F, {
X, Undef});
17099 case SystemZ::BI__builtin_s390_vctzb:
17100 case SystemZ::BI__builtin_s390_vctzh:
17101 case SystemZ::BI__builtin_s390_vctzf:
17102 case SystemZ::BI__builtin_s390_vctzg: {
17105 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
17107 return Builder.CreateCall(F, {
X, Undef});
17110 case SystemZ::BI__builtin_s390_vfsqsb:
17111 case SystemZ::BI__builtin_s390_vfsqdb: {
17114 if (Builder.getIsFPConstrained()) {
17115 Function *F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_sqrt, ResultType);
17116 return Builder.CreateConstrainedFPCall(F, {
X });
17119 return Builder.CreateCall(F,
X);
17122 case SystemZ::BI__builtin_s390_vfmasb:
17123 case SystemZ::BI__builtin_s390_vfmadb: {
17128 if (Builder.getIsFPConstrained()) {
17129 Function *F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
17130 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
17133 return Builder.CreateCall(F, {
X, Y, Z});
17136 case SystemZ::BI__builtin_s390_vfmssb:
17137 case SystemZ::BI__builtin_s390_vfmsdb: {
17142 if (Builder.getIsFPConstrained()) {
17143 Function *F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
17144 return Builder.CreateConstrainedFPCall(F, {
X, Y, Builder.CreateFNeg(Z,
"neg")});
17147 return Builder.CreateCall(F, {
X, Y, Builder.CreateFNeg(Z,
"neg")});
17150 case SystemZ::BI__builtin_s390_vfnmasb:
17151 case SystemZ::BI__builtin_s390_vfnmadb: {
17156 if (Builder.getIsFPConstrained()) {
17157 Function *F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
17158 return Builder.CreateFNeg(Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
17161 return Builder.CreateFNeg(Builder.CreateCall(F, {X, Y, Z}),
"neg");
17164 case SystemZ::BI__builtin_s390_vfnmssb:
17165 case SystemZ::BI__builtin_s390_vfnmsdb: {
17170 if (Builder.getIsFPConstrained()) {
17171 Function *F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
17172 Value *NegZ = Builder.CreateFNeg(Z,
"sub");
17173 return Builder.CreateFNeg(Builder.CreateConstrainedFPCall(F, {X, Y, NegZ}));
17176 Value *NegZ = Builder.CreateFNeg(Z,
"neg");
17177 return Builder.CreateFNeg(Builder.CreateCall(F, {X, Y, NegZ}));
17180 case SystemZ::BI__builtin_s390_vflpsb:
17181 case SystemZ::BI__builtin_s390_vflpdb: {
17185 return Builder.CreateCall(F,
X);
17187 case SystemZ::BI__builtin_s390_vflnsb:
17188 case SystemZ::BI__builtin_s390_vflndb: {
17192 return Builder.CreateFNeg(Builder.CreateCall(F,
X),
"neg");
17194 case SystemZ::BI__builtin_s390_vfisb:
17195 case SystemZ::BI__builtin_s390_vfidb: {
17203 Intrinsic::ID ID = Intrinsic::not_intrinsic;
17205 switch (M4.getZExtValue()) {
17208 switch (M5.getZExtValue()) {
17210 case 0: ID = Intrinsic::rint;
17211 CI = Intrinsic::experimental_constrained_rint;
break;
17215 switch (M5.getZExtValue()) {
17217 case 0: ID = Intrinsic::nearbyint;
17218 CI = Intrinsic::experimental_constrained_nearbyint;
break;
17219 case 1: ID = Intrinsic::round;
17220 CI = Intrinsic::experimental_constrained_round;
break;
17221 case 5: ID = Intrinsic::trunc;
17222 CI = Intrinsic::experimental_constrained_trunc;
break;
17223 case 6: ID = Intrinsic::ceil;
17224 CI = Intrinsic::experimental_constrained_ceil;
break;
17225 case 7: ID = Intrinsic::floor;
17226 CI = Intrinsic::experimental_constrained_floor;
break;
17230 if (ID != Intrinsic::not_intrinsic) {
17231 if (Builder.getIsFPConstrained()) {
17233 return Builder.CreateConstrainedFPCall(F,
X);
17236 return Builder.CreateCall(F,
X);
17239 switch (BuiltinID) {
17240 case SystemZ::BI__builtin_s390_vfisb: ID = Intrinsic::s390_vfisb;
break;
17241 case SystemZ::BI__builtin_s390_vfidb: ID = Intrinsic::s390_vfidb;
break;
17242 default: llvm_unreachable(
"Unknown BuiltinID");
17247 return Builder.CreateCall(F, {
X, M4Value, M5Value});
17249 case SystemZ::BI__builtin_s390_vfmaxsb:
17250 case SystemZ::BI__builtin_s390_vfmaxdb: {
17258 Intrinsic::ID ID = Intrinsic::not_intrinsic;
17260 switch (M4.getZExtValue()) {
17262 case 4: ID = Intrinsic::maxnum;
17263 CI = Intrinsic::experimental_constrained_maxnum;
break;
17265 if (ID != Intrinsic::not_intrinsic) {
17266 if (Builder.getIsFPConstrained()) {
17268 return Builder.CreateConstrainedFPCall(F, {
X, Y});
17271 return Builder.CreateCall(F, {
X, Y});
17274 switch (BuiltinID) {
17275 case SystemZ::BI__builtin_s390_vfmaxsb: ID = Intrinsic::s390_vfmaxsb;
break;
17276 case SystemZ::BI__builtin_s390_vfmaxdb: ID = Intrinsic::s390_vfmaxdb;
break;
17277 default: llvm_unreachable(
"Unknown BuiltinID");
17281 return Builder.CreateCall(F, {
X, Y, M4Value});
17283 case SystemZ::BI__builtin_s390_vfminsb:
17284 case SystemZ::BI__builtin_s390_vfmindb: {
17292 Intrinsic::ID ID = Intrinsic::not_intrinsic;
17294 switch (M4.getZExtValue()) {
17296 case 4: ID = Intrinsic::minnum;
17297 CI = Intrinsic::experimental_constrained_minnum;
break;
17299 if (ID != Intrinsic::not_intrinsic) {
17300 if (Builder.getIsFPConstrained()) {
17302 return Builder.CreateConstrainedFPCall(F, {
X, Y});
17305 return Builder.CreateCall(F, {
X, Y});
17308 switch (BuiltinID) {
17309 case SystemZ::BI__builtin_s390_vfminsb: ID = Intrinsic::s390_vfminsb;
break;
17310 case SystemZ::BI__builtin_s390_vfmindb: ID = Intrinsic::s390_vfmindb;
break;
17311 default: llvm_unreachable(
"Unknown BuiltinID");
17315 return Builder.CreateCall(F, {
X, Y, M4Value});
17318 case SystemZ::BI__builtin_s390_vlbrh:
17319 case SystemZ::BI__builtin_s390_vlbrf:
17320 case SystemZ::BI__builtin_s390_vlbrg: {
17324 return Builder.CreateCall(F,
X);
17329#define INTRINSIC_WITH_CC(NAME) \
17330 case SystemZ::BI__builtin_##NAME: \
17331 return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E)
17410#undef INTRINSIC_WITH_CC
17419struct NVPTXMmaLdstInfo {
17420 unsigned NumResults;
17426#define MMA_INTR(geom_op_type, layout) \
17427 Intrinsic::nvvm_wmma_##geom_op_type##_##layout##_stride
17428#define MMA_LDST(n, geom_op_type) \
17429 { n, MMA_INTR(geom_op_type, col), MMA_INTR(geom_op_type, row) }
17431static NVPTXMmaLdstInfo getNVPTXMmaLdstInfo(
unsigned BuiltinID) {
17432 switch (BuiltinID) {
17434 case NVPTX::BI__hmma_m16n16k16_ld_a:
17435 return MMA_LDST(8, m16n16k16_load_a_f16);
17436 case NVPTX::BI__hmma_m16n16k16_ld_b:
17437 return MMA_LDST(8, m16n16k16_load_b_f16);
17438 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
17439 return MMA_LDST(4, m16n16k16_load_c_f16);
17440 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
17441 return MMA_LDST(8, m16n16k16_load_c_f32);
17442 case NVPTX::BI__hmma_m32n8k16_ld_a:
17443 return MMA_LDST(8, m32n8k16_load_a_f16);
17444 case NVPTX::BI__hmma_m32n8k16_ld_b:
17445 return MMA_LDST(8, m32n8k16_load_b_f16);
17446 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
17447 return MMA_LDST(4, m32n8k16_load_c_f16);
17448 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
17449 return MMA_LDST(8, m32n8k16_load_c_f32);
17450 case NVPTX::BI__hmma_m8n32k16_ld_a:
17451 return MMA_LDST(8, m8n32k16_load_a_f16);
17452 case NVPTX::BI__hmma_m8n32k16_ld_b:
17453 return MMA_LDST(8, m8n32k16_load_b_f16);
17454 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
17455 return MMA_LDST(4, m8n32k16_load_c_f16);
17456 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
17457 return MMA_LDST(8, m8n32k16_load_c_f32);
17460 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
17461 return MMA_LDST(2, m16n16k16_load_a_s8);
17462 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
17463 return MMA_LDST(2, m16n16k16_load_a_u8);
17464 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
17465 return MMA_LDST(2, m16n16k16_load_b_s8);
17466 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
17467 return MMA_LDST(2, m16n16k16_load_b_u8);
17468 case NVPTX::BI__imma_m16n16k16_ld_c:
17469 return MMA_LDST(8, m16n16k16_load_c_s32);
17470 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
17471 return MMA_LDST(4, m32n8k16_load_a_s8);
17472 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
17473 return MMA_LDST(4, m32n8k16_load_a_u8);
17474 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
17475 return MMA_LDST(1, m32n8k16_load_b_s8);
17476 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
17477 return MMA_LDST(1, m32n8k16_load_b_u8);
17478 case NVPTX::BI__imma_m32n8k16_ld_c:
17479 return MMA_LDST(8, m32n8k16_load_c_s32);
17480 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
17481 return MMA_LDST(1, m8n32k16_load_a_s8);
17482 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
17483 return MMA_LDST(1, m8n32k16_load_a_u8);
17484 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
17485 return MMA_LDST(4, m8n32k16_load_b_s8);
17486 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
17487 return MMA_LDST(4, m8n32k16_load_b_u8);
17488 case NVPTX::BI__imma_m8n32k16_ld_c:
17489 return MMA_LDST(8, m8n32k16_load_c_s32);
17493 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
17494 return {1, 0,
MMA_INTR(m8n8k32_load_a_s4, row)};
17495 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
17496 return {1, 0,
MMA_INTR(m8n8k32_load_a_u4, row)};
17497 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
17498 return {1,
MMA_INTR(m8n8k32_load_b_s4, col), 0};
17499 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
17500 return {1,
MMA_INTR(m8n8k32_load_b_u4, col), 0};
17501 case NVPTX::BI__imma_m8n8k32_ld_c:
17502 return MMA_LDST(2, m8n8k32_load_c_s32);
17503 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
17504 return {1, 0,
MMA_INTR(m8n8k128_load_a_b1, row)};
17505 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
17506 return {1,
MMA_INTR(m8n8k128_load_b_b1, col), 0};
17507 case NVPTX::BI__bmma_m8n8k128_ld_c:
17508 return MMA_LDST(2, m8n8k128_load_c_s32);
17511 case NVPTX::BI__dmma_m8n8k4_ld_a:
17512 return MMA_LDST(1, m8n8k4_load_a_f64);
17513 case NVPTX::BI__dmma_m8n8k4_ld_b:
17514 return MMA_LDST(1, m8n8k4_load_b_f64);
17515 case NVPTX::BI__dmma_m8n8k4_ld_c:
17516 return MMA_LDST(2, m8n8k4_load_c_f64);
17519 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
17520 return MMA_LDST(4, m16n16k16_load_a_bf16);
17521 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
17522 return MMA_LDST(4, m16n16k16_load_b_bf16);
17523 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
17524 return MMA_LDST(2, m8n32k16_load_a_bf16);
17525 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
17526 return MMA_LDST(8, m8n32k16_load_b_bf16);
17527 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
17528 return MMA_LDST(8, m32n8k16_load_a_bf16);
17529 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
17530 return MMA_LDST(2, m32n8k16_load_b_bf16);
17531 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
17532 return MMA_LDST(4, m16n16k8_load_a_tf32);
17533 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
17534 return MMA_LDST(4, m16n16k8_load_b_tf32);
17535 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
17536 return MMA_LDST(8, m16n16k8_load_c_f32);
17542 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
17543 return MMA_LDST(4, m16n16k16_store_d_f16);
17544 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
17545 return MMA_LDST(8, m16n16k16_store_d_f32);
17546 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
17547 return MMA_LDST(4, m32n8k16_store_d_f16);
17548 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
17549 return MMA_LDST(8, m32n8k16_store_d_f32);
17550 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
17551 return MMA_LDST(4, m8n32k16_store_d_f16);
17552 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
17553 return MMA_LDST(8, m8n32k16_store_d_f32);
17558 case NVPTX::BI__imma_m16n16k16_st_c_i32:
17559 return MMA_LDST(8, m16n16k16_store_d_s32);
17560 case NVPTX::BI__imma_m32n8k16_st_c_i32:
17561 return MMA_LDST(8, m32n8k16_store_d_s32);
17562 case NVPTX::BI__imma_m8n32k16_st_c_i32:
17563 return MMA_LDST(8, m8n32k16_store_d_s32);
17564 case NVPTX::BI__imma_m8n8k32_st_c_i32:
17565 return MMA_LDST(2, m8n8k32_store_d_s32);
17566 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
17567 return MMA_LDST(2, m8n8k128_store_d_s32);
17570 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
17571 return MMA_LDST(2, m8n8k4_store_d_f64);
17574 case NVPTX::BI__mma_m16n16k8_st_c_f32:
17575 return MMA_LDST(8, m16n16k8_store_d_f32);
17578 llvm_unreachable(
"Unknown MMA builtin");
17585struct NVPTXMmaInfo {
17594 std::array<unsigned, 8> Variants;
17596 unsigned getMMAIntrinsic(
int Layout,
bool Satf) {
17597 unsigned Index = Layout + 4 * Satf;
17598 if (Index >= Variants.size())
17600 return Variants[Index];
17606static NVPTXMmaInfo getNVPTXMmaInfo(
unsigned BuiltinID) {
17608#define MMA_VARIANTS(geom, type) \
17609 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type, \
17610 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
17611 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type, \
17612 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type
17613#define MMA_SATF_VARIANTS(geom, type) \
17614 MMA_VARIANTS(geom, type), \
17615 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type##_satfinite, \
17616 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
17617 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type##_satfinite, \
17618 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type##_satfinite
17620#define MMA_VARIANTS_I4(geom, type) \
17622 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
17626 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
17630#define MMA_VARIANTS_B1_XOR(geom, type) \
17632 Intrinsic::nvvm_wmma_##geom##_mma_xor_popc_row_col_##type, \
17639#define MMA_VARIANTS_B1_AND(geom, type) \
17641 Intrinsic::nvvm_wmma_##geom##_mma_and_popc_row_col_##type, \
17649 switch (BuiltinID) {
17653 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
17655 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
17657 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
17659 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
17661 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
17663 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
17665 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
17667 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
17669 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
17671 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
17673 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
17675 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
17679 case NVPTX::BI__imma_m16n16k16_mma_s8:
17681 case NVPTX::BI__imma_m16n16k16_mma_u8:
17683 case NVPTX::BI__imma_m32n8k16_mma_s8:
17685 case NVPTX::BI__imma_m32n8k16_mma_u8:
17687 case NVPTX::BI__imma_m8n32k16_mma_s8:
17689 case NVPTX::BI__imma_m8n32k16_mma_u8:
17693 case NVPTX::BI__imma_m8n8k32_mma_s4:
17695 case NVPTX::BI__imma_m8n8k32_mma_u4:
17697 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
17699 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
17703 case NVPTX::BI__dmma_m8n8k4_mma_f64:
17707 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
17708 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k16, bf16)}}};
17709 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
17710 return {2, 8, 8, 8, {{
MMA_VARIANTS(m8n32k16, bf16)}}};
17711 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
17712 return {8, 2, 8, 8, {{
MMA_VARIANTS(m32n8k16, bf16)}}};
17713 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
17714 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k8, tf32)}}};
17716 llvm_unreachable(
"Unexpected builtin ID.");
17719#undef MMA_SATF_VARIANTS
17720#undef MMA_VARIANTS_I4
17721#undef MMA_VARIANTS_B1_AND
17722#undef MMA_VARIANTS_B1_XOR
17729 auto MakeLdg = [&](
unsigned IntrinsicID) {
17734 return Builder.CreateCall(
17736 {Ptr, ConstantInt::get(Builder.getInt32Ty(), Align.getQuantity())});
17738 auto MakeScopedAtomic = [&](
unsigned IntrinsicID) {
17740 llvm::Type *ElemTy =
17742 return Builder.CreateCall(
17744 {Ptr, EmitScalarExpr(E->getArg(1))});
17746 switch (BuiltinID) {
17747 case NVPTX::BI__nvvm_atom_add_gen_i:
17748 case NVPTX::BI__nvvm_atom_add_gen_l:
17749 case NVPTX::BI__nvvm_atom_add_gen_ll:
17752 case NVPTX::BI__nvvm_atom_sub_gen_i:
17753 case NVPTX::BI__nvvm_atom_sub_gen_l:
17754 case NVPTX::BI__nvvm_atom_sub_gen_ll:
17757 case NVPTX::BI__nvvm_atom_and_gen_i:
17758 case NVPTX::BI__nvvm_atom_and_gen_l:
17759 case NVPTX::BI__nvvm_atom_and_gen_ll:
17762 case NVPTX::BI__nvvm_atom_or_gen_i:
17763 case NVPTX::BI__nvvm_atom_or_gen_l:
17764 case NVPTX::BI__nvvm_atom_or_gen_ll:
17767 case NVPTX::BI__nvvm_atom_xor_gen_i:
17768 case NVPTX::BI__nvvm_atom_xor_gen_l:
17769 case NVPTX::BI__nvvm_atom_xor_gen_ll:
17772 case NVPTX::BI__nvvm_atom_xchg_gen_i:
17773 case NVPTX::BI__nvvm_atom_xchg_gen_l:
17774 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
17777 case NVPTX::BI__nvvm_atom_max_gen_i:
17778 case NVPTX::BI__nvvm_atom_max_gen_l:
17779 case NVPTX::BI__nvvm_atom_max_gen_ll:
17782 case NVPTX::BI__nvvm_atom_max_gen_ui:
17783 case NVPTX::BI__nvvm_atom_max_gen_ul:
17784 case NVPTX::BI__nvvm_atom_max_gen_ull:
17787 case NVPTX::BI__nvvm_atom_min_gen_i:
17788 case NVPTX::BI__nvvm_atom_min_gen_l:
17789 case NVPTX::BI__nvvm_atom_min_gen_ll:
17792 case NVPTX::BI__nvvm_atom_min_gen_ui:
17793 case NVPTX::BI__nvvm_atom_min_gen_ul:
17794 case NVPTX::BI__nvvm_atom_min_gen_ull:
17797 case NVPTX::BI__nvvm_atom_cas_gen_i:
17798 case NVPTX::BI__nvvm_atom_cas_gen_l:
17799 case NVPTX::BI__nvvm_atom_cas_gen_ll:
17804 case NVPTX::BI__nvvm_atom_add_gen_f:
17805 case NVPTX::BI__nvvm_atom_add_gen_d: {
17808 return Builder.CreateAtomicRMW(llvm::AtomicRMWInst::FAdd, Ptr, Val,
17809 AtomicOrdering::SequentiallyConsistent);
17812 case NVPTX::BI__nvvm_atom_inc_gen_ui: {
17815 Function *FnALI32 =
17816 CGM.
getIntrinsic(Intrinsic::nvvm_atomic_load_inc_32, Ptr->getType());
17817 return Builder.CreateCall(FnALI32, {Ptr, Val});
17820 case NVPTX::BI__nvvm_atom_dec_gen_ui: {
17823 Function *FnALD32 =
17824 CGM.
getIntrinsic(Intrinsic::nvvm_atomic_load_dec_32, Ptr->getType());
17825 return Builder.CreateCall(FnALD32, {Ptr, Val});
17828 case NVPTX::BI__nvvm_ldg_c:
17829 case NVPTX::BI__nvvm_ldg_c2:
17830 case NVPTX::BI__nvvm_ldg_c4:
17831 case NVPTX::BI__nvvm_ldg_s:
17832 case NVPTX::BI__nvvm_ldg_s2:
17833 case NVPTX::BI__nvvm_ldg_s4:
17834 case NVPTX::BI__nvvm_ldg_i:
17835 case NVPTX::BI__nvvm_ldg_i2:
17836 case NVPTX::BI__nvvm_ldg_i4:
17837 case NVPTX::BI__nvvm_ldg_l:
17838 case NVPTX::BI__nvvm_ldg_ll:
17839 case NVPTX::BI__nvvm_ldg_ll2:
17840 case NVPTX::BI__nvvm_ldg_uc:
17841 case NVPTX::BI__nvvm_ldg_uc2:
17842 case NVPTX::BI__nvvm_ldg_uc4:
17843 case NVPTX::BI__nvvm_ldg_us:
17844 case NVPTX::BI__nvvm_ldg_us2:
17845 case NVPTX::BI__nvvm_ldg_us4:
17846 case NVPTX::BI__nvvm_ldg_ui:
17847 case NVPTX::BI__nvvm_ldg_ui2:
17848 case NVPTX::BI__nvvm_ldg_ui4:
17849 case NVPTX::BI__nvvm_ldg_ul:
17850 case NVPTX::BI__nvvm_ldg_ull:
17851 case NVPTX::BI__nvvm_ldg_ull2:
17855 return MakeLdg(Intrinsic::nvvm_ldg_global_i);
17856 case NVPTX::BI__nvvm_ldg_f:
17857 case NVPTX::BI__nvvm_ldg_f2:
17858 case NVPTX::BI__nvvm_ldg_f4:
17859 case NVPTX::BI__nvvm_ldg_d:
17860 case NVPTX::BI__nvvm_ldg_d2:
17861 return MakeLdg(Intrinsic::nvvm_ldg_global_f);
17863 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
17864 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
17865 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
17866 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta);
17867 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
17868 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
17869 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
17870 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys);
17871 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
17872 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
17873 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta);
17874 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
17875 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
17876 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys);
17877 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
17878 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
17879 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
17880 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta);
17881 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
17882 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
17883 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
17884 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys);
17885 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
17886 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
17887 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
17888 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
17889 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
17890 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
17891 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta);
17892 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
17893 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
17894 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
17895 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
17896 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
17897 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
17898 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys);
17899 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
17900 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
17901 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
17902 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
17903 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
17904 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
17905 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta);
17906 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
17907 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
17908 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
17909 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
17910 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
17911 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
17912 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys);
17913 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
17914 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta);
17915 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
17916 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta);
17917 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
17918 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys);
17919 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
17920 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys);
17921 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
17922 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
17923 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
17924 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta);
17925 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
17926 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
17927 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
17928 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys);
17929 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
17930 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
17931 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
17932 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta);
17933 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
17934 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
17935 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
17936 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys);
17937 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
17938 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
17939 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
17940 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta);
17941 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
17942 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
17943 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
17944 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys);
17945 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
17946 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
17947 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: {
17949 llvm::Type *ElemTy =
17951 return Builder.CreateCall(
17953 Intrinsic::nvvm_atomic_cas_gen_i_cta, {ElemTy, Ptr->getType()}),
17954 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
17956 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
17957 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
17958 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: {
17960 llvm::Type *ElemTy =
17962 return Builder.CreateCall(
17964 Intrinsic::nvvm_atomic_cas_gen_i_sys, {ElemTy, Ptr->getType()}),
17965 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
17967 case NVPTX::BI__nvvm_match_all_sync_i32p:
17968 case NVPTX::BI__nvvm_match_all_sync_i64p: {
17972 Value *ResultPair = Builder.CreateCall(
17974 ? Intrinsic::nvvm_match_all_sync_i32p
17975 : Intrinsic::nvvm_match_all_sync_i64p),
17977 Value *Pred = Builder.CreateZExt(Builder.CreateExtractValue(ResultPair, 1),
17979 Builder.CreateStore(Pred, PredOutPtr);
17980 return Builder.CreateExtractValue(ResultPair, 0);
17984 case NVPTX::BI__hmma_m16n16k16_ld_a:
17985 case NVPTX::BI__hmma_m16n16k16_ld_b:
17986 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
17987 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
17988 case NVPTX::BI__hmma_m32n8k16_ld_a:
17989 case NVPTX::BI__hmma_m32n8k16_ld_b:
17990 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
17991 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
17992 case NVPTX::BI__hmma_m8n32k16_ld_a:
17993 case NVPTX::BI__hmma_m8n32k16_ld_b:
17994 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
17995 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
17997 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
17998 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
17999 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
18000 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
18001 case NVPTX::BI__imma_m16n16k16_ld_c:
18002 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
18003 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
18004 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
18005 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
18006 case NVPTX::BI__imma_m32n8k16_ld_c:
18007 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
18008 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
18009 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
18010 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
18011 case NVPTX::BI__imma_m8n32k16_ld_c:
18013 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
18014 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
18015 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
18016 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
18017 case NVPTX::BI__imma_m8n8k32_ld_c:
18018 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
18019 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
18020 case NVPTX::BI__bmma_m8n8k128_ld_c:
18022 case NVPTX::BI__dmma_m8n8k4_ld_a:
18023 case NVPTX::BI__dmma_m8n8k4_ld_b:
18024 case NVPTX::BI__dmma_m8n8k4_ld_c:
18026 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
18027 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
18028 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
18029 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
18030 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
18031 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
18032 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
18033 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
18034 case NVPTX::BI__mma_tf32_m16n16k8_ld_c: {
18040 if (!isColMajorArg)
18042 bool isColMajor = isColMajorArg->getSExtValue();
18043 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
18044 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
18049 Builder.CreateCall(
CGM.
getIntrinsic(IID, Src->getType()), {Src, Ldm});
18052 assert(II.NumResults);
18053 if (II.NumResults == 1) {
18057 for (
unsigned i = 0; i < II.NumResults; ++i) {
18058 Builder.CreateAlignedStore(
18059 Builder.CreateBitCast(Builder.CreateExtractValue(
Result, i),
18062 llvm::ConstantInt::get(
IntTy, i)),
18069 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
18070 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
18071 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
18072 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
18073 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
18074 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
18075 case NVPTX::BI__imma_m16n16k16_st_c_i32:
18076 case NVPTX::BI__imma_m32n8k16_st_c_i32:
18077 case NVPTX::BI__imma_m8n32k16_st_c_i32:
18078 case NVPTX::BI__imma_m8n8k32_st_c_i32:
18079 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
18080 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
18081 case NVPTX::BI__mma_m16n16k8_st_c_f32: {
18087 if (!isColMajorArg)
18089 bool isColMajor = isColMajorArg->getSExtValue();
18090 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
18091 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
18094 Function *Intrinsic =
18096 llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(1);
18098 for (
unsigned i = 0; i < II.NumResults; ++i) {
18099 Value *
V = Builder.CreateAlignedLoad(
18102 llvm::ConstantInt::get(
IntTy, i)),
18104 Values.push_back(Builder.CreateBitCast(
V, ParamType));
18106 Values.push_back(Ldm);
18107 Value *
Result = Builder.CreateCall(Intrinsic, Values);
18113 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
18114 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
18115 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
18116 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
18117 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
18118 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
18119 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
18120 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
18121 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
18122 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
18123 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
18124 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
18125 case NVPTX::BI__imma_m16n16k16_mma_s8:
18126 case NVPTX::BI__imma_m16n16k16_mma_u8:
18127 case NVPTX::BI__imma_m32n8k16_mma_s8:
18128 case NVPTX::BI__imma_m32n8k16_mma_u8:
18129 case NVPTX::BI__imma_m8n32k16_mma_s8:
18130 case NVPTX::BI__imma_m8n32k16_mma_u8:
18131 case NVPTX::BI__imma_m8n8k32_mma_s4:
18132 case NVPTX::BI__imma_m8n8k32_mma_u4:
18133 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
18134 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
18135 case NVPTX::BI__dmma_m8n8k4_mma_f64:
18136 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
18137 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
18138 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
18139 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32: {
18148 int Layout = LayoutArg->getSExtValue();
18149 if (Layout < 0 || Layout > 3)
18151 llvm::APSInt SatfArg;
18152 if (BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1 ||
18153 BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1)
18157 SatfArg = *OptSatfArg;
18160 bool Satf = SatfArg.getSExtValue();
18161 NVPTXMmaInfo MI = getNVPTXMmaInfo(BuiltinID);
18162 unsigned IID = MI.getMMAIntrinsic(Layout, Satf);
18168 llvm::Type *AType = Intrinsic->getFunctionType()->getParamType(0);
18170 for (
unsigned i = 0; i < MI.NumEltsA; ++i) {
18171 Value *
V = Builder.CreateAlignedLoad(
18174 llvm::ConstantInt::get(
IntTy, i)),
18176 Values.push_back(Builder.CreateBitCast(
V, AType));
18179 llvm::Type *BType = Intrinsic->getFunctionType()->getParamType(MI.NumEltsA);
18180 for (
unsigned i = 0; i < MI.NumEltsB; ++i) {
18181 Value *
V = Builder.CreateAlignedLoad(
18184 llvm::ConstantInt::get(
IntTy, i)),
18186 Values.push_back(Builder.CreateBitCast(
V, BType));
18189 llvm::Type *CType =
18190 Intrinsic->getFunctionType()->getParamType(MI.NumEltsA + MI.NumEltsB);
18191 for (
unsigned i = 0; i < MI.NumEltsC; ++i) {
18192 Value *
V = Builder.CreateAlignedLoad(
18195 llvm::ConstantInt::get(
IntTy, i)),
18197 Values.push_back(Builder.CreateBitCast(
V, CType));
18199 Value *
Result = Builder.CreateCall(Intrinsic, Values);
18201 for (
unsigned i = 0; i < MI.NumEltsD; ++i)
18202 Builder.CreateAlignedStore(
18203 Builder.CreateBitCast(Builder.CreateExtractValue(
Result, i), DType),
18205 llvm::ConstantInt::get(
IntTy, i)),
18215struct BuiltinAlignArgs {
18216 llvm::Value *Src =
nullptr;
18217 llvm::Type *SrcType =
nullptr;
18218 llvm::Value *Alignment =
nullptr;
18219 llvm::Value *Mask =
nullptr;
18220 llvm::IntegerType *IntType =
nullptr;
18228 SrcType = Src->getType();
18229 if (SrcType->isPointerTy()) {
18230 IntType = IntegerType::get(
18234 assert(SrcType->isIntegerTy());
18238 Alignment = CGF.
Builder.CreateZExtOrTrunc(Alignment, IntType,
"alignment");
18239 auto *One = llvm::ConstantInt::get(IntType, 1);
18240 Mask = CGF.
Builder.CreateSub(Alignment, One,
"mask");
18247 BuiltinAlignArgs Args(E, *
this);
18248 llvm::Value *SrcAddress = Args.Src;
18249 if (Args.SrcType->isPointerTy())
18251 Builder.CreateBitOrPointerCast(Args.Src, Args.IntType,
"src_addr");
18253 Builder.CreateAnd(SrcAddress, Args.Mask,
"set_bits"),
18254 llvm::Constant::getNullValue(Args.IntType),
"is_aligned"));
18262 BuiltinAlignArgs Args(E, *
this);
18263 llvm::Value *SrcAddr = Args.Src;
18264 if (Args.Src->getType()->isPointerTy())
18265 SrcAddr = Builder.CreatePtrToInt(Args.Src, Args.IntType,
"intptr");
18266 llvm::Value *SrcForMask = SrcAddr;
18272 SrcForMask = Builder.CreateAdd(SrcForMask, Args.Mask,
"over_boundary");
18275 llvm::Value *InvertedMask = Builder.CreateNot(Args.Mask,
"inverted_mask");
18277 Builder.CreateAnd(SrcForMask, InvertedMask,
"aligned_result");
18278 if (Args.Src->getType()->isPointerTy()) {
18283 Result->setName(
"aligned_intptr");
18284 llvm::Value *Difference = Builder.CreateSub(
Result, SrcAddr,
"diff");
18295 Result = Builder.CreatePointerCast(
Result, Args.SrcType);
18300 assert(
Result->getType() == Args.SrcType);
18306 switch (BuiltinID) {
18307 case WebAssembly::BI__builtin_wasm_memory_size: {
18312 return Builder.CreateCall(Callee, I);
18314 case WebAssembly::BI__builtin_wasm_memory_grow: {
18320 return Builder.CreateCall(Callee, Args);
18322 case WebAssembly::BI__builtin_wasm_tls_size: {
18325 return Builder.CreateCall(Callee);
18327 case WebAssembly::BI__builtin_wasm_tls_align: {
18330 return Builder.CreateCall(Callee);
18332 case WebAssembly::BI__builtin_wasm_tls_base: {
18334 return Builder.CreateCall(Callee);
18336 case WebAssembly::BI__builtin_wasm_throw: {
18340 return Builder.CreateCall(Callee, {Tag, Obj});
18342 case WebAssembly::BI__builtin_wasm_rethrow: {
18344 return Builder.CreateCall(Callee);
18346 case WebAssembly::BI__builtin_wasm_memory_atomic_wait32: {
18351 return Builder.CreateCall(Callee, {Addr,
Expected, Timeout});
18353 case WebAssembly::BI__builtin_wasm_memory_atomic_wait64: {
18358 return Builder.CreateCall(Callee, {Addr,
Expected, Timeout});
18360 case WebAssembly::BI__builtin_wasm_memory_atomic_notify: {
18364 return Builder.CreateCall(Callee, {Addr, Count});
18366 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f32:
18367 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f64:
18368 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f32:
18369 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f64: {
18373 CGM.
getIntrinsic(Intrinsic::wasm_trunc_signed, {ResT, Src->getType()});
18374 return Builder.CreateCall(Callee, {Src});
18376 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f32:
18377 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f64:
18378 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f32:
18379 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f64: {
18383 {ResT, Src->getType()});
18384 return Builder.CreateCall(Callee, {Src});
18386 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f32:
18387 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f64:
18388 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f32:
18389 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f64:
18390 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32x4_f32x4: {
18395 return Builder.CreateCall(Callee, {Src});
18397 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f32:
18398 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f64:
18399 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f32:
18400 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f64:
18401 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32x4_f32x4: {
18406 return Builder.CreateCall(Callee, {Src});
18408 case WebAssembly::BI__builtin_wasm_min_f32:
18409 case WebAssembly::BI__builtin_wasm_min_f64:
18410 case WebAssembly::BI__builtin_wasm_min_f32x4:
18411 case WebAssembly::BI__builtin_wasm_min_f64x2: {
18416 return Builder.CreateCall(Callee, {LHS, RHS});
18418 case WebAssembly::BI__builtin_wasm_max_f32:
18419 case WebAssembly::BI__builtin_wasm_max_f64:
18420 case WebAssembly::BI__builtin_wasm_max_f32x4:
18421 case WebAssembly::BI__builtin_wasm_max_f64x2: {
18426 return Builder.CreateCall(Callee, {LHS, RHS});
18428 case WebAssembly::BI__builtin_wasm_pmin_f32x4:
18429 case WebAssembly::BI__builtin_wasm_pmin_f64x2: {
18434 return Builder.CreateCall(Callee, {LHS, RHS});
18436 case WebAssembly::BI__builtin_wasm_pmax_f32x4:
18437 case WebAssembly::BI__builtin_wasm_pmax_f64x2: {
18442 return Builder.CreateCall(Callee, {LHS, RHS});
18444 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
18445 case WebAssembly::BI__builtin_wasm_floor_f32x4:
18446 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
18447 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
18448 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
18449 case WebAssembly::BI__builtin_wasm_floor_f64x2:
18450 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
18451 case WebAssembly::BI__builtin_wasm_nearest_f64x2: {
18453 switch (BuiltinID) {
18454 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
18455 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
18456 IntNo = Intrinsic::ceil;
18458 case WebAssembly::BI__builtin_wasm_floor_f32x4:
18459 case WebAssembly::BI__builtin_wasm_floor_f64x2:
18460 IntNo = Intrinsic::floor;
18462 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
18463 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
18464 IntNo = Intrinsic::trunc;
18466 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
18467 case WebAssembly::BI__builtin_wasm_nearest_f64x2:
18468 IntNo = Intrinsic::nearbyint;
18471 llvm_unreachable(
"unexpected builtin ID");
18475 return Builder.CreateCall(Callee,
Value);
18477 case WebAssembly::BI__builtin_wasm_swizzle_i8x16: {
18481 return Builder.CreateCall(Callee, {Src, Indices});
18483 case WebAssembly::BI__builtin_wasm_add_sat_s_i8x16:
18484 case WebAssembly::BI__builtin_wasm_add_sat_u_i8x16:
18485 case WebAssembly::BI__builtin_wasm_add_sat_s_i16x8:
18486 case WebAssembly::BI__builtin_wasm_add_sat_u_i16x8:
18487 case WebAssembly::BI__builtin_wasm_sub_sat_s_i8x16:
18488 case WebAssembly::BI__builtin_wasm_sub_sat_u_i8x16:
18489 case WebAssembly::BI__builtin_wasm_sub_sat_s_i16x8:
18490 case WebAssembly::BI__builtin_wasm_sub_sat_u_i16x8: {
18492 switch (BuiltinID) {
18493 case WebAssembly::BI__builtin_wasm_add_sat_s_i8x16:
18494 case WebAssembly::BI__builtin_wasm_add_sat_s_i16x8:
18495 IntNo = Intrinsic::sadd_sat;
18497 case WebAssembly::BI__builtin_wasm_add_sat_u_i8x16:
18498 case WebAssembly::BI__builtin_wasm_add_sat_u_i16x8:
18499 IntNo = Intrinsic::uadd_sat;
18501 case WebAssembly::BI__builtin_wasm_sub_sat_s_i8x16:
18502 case WebAssembly::BI__builtin_wasm_sub_sat_s_i16x8:
18503 IntNo = Intrinsic::wasm_sub_sat_signed;
18505 case WebAssembly::BI__builtin_wasm_sub_sat_u_i8x16:
18506 case WebAssembly::BI__builtin_wasm_sub_sat_u_i16x8:
18507 IntNo = Intrinsic::wasm_sub_sat_unsigned;
18510 llvm_unreachable(
"unexpected builtin ID");
18515 return Builder.CreateCall(Callee, {LHS, RHS});
18517 case WebAssembly::BI__builtin_wasm_abs_i8x16:
18518 case WebAssembly::BI__builtin_wasm_abs_i16x8:
18519 case WebAssembly::BI__builtin_wasm_abs_i32x4:
18520 case WebAssembly::BI__builtin_wasm_abs_i64x2: {
18522 Value *Neg = Builder.CreateNeg(Vec,
"neg");
18523 Constant *
Zero = llvm::Constant::getNullValue(Vec->getType());
18524 Value *ICmp = Builder.CreateICmpSLT(Vec, Zero,
"abscond");
18525 return Builder.CreateSelect(ICmp, Neg, Vec,
"abs");
18527 case WebAssembly::BI__builtin_wasm_min_s_i8x16:
18528 case WebAssembly::BI__builtin_wasm_min_u_i8x16:
18529 case WebAssembly::BI__builtin_wasm_max_s_i8x16:
18530 case WebAssembly::BI__builtin_wasm_max_u_i8x16:
18531 case WebAssembly::BI__builtin_wasm_min_s_i16x8:
18532 case WebAssembly::BI__builtin_wasm_min_u_i16x8:
18533 case WebAssembly::BI__builtin_wasm_max_s_i16x8:
18534 case WebAssembly::BI__builtin_wasm_max_u_i16x8:
18535 case WebAssembly::BI__builtin_wasm_min_s_i32x4:
18536 case WebAssembly::BI__builtin_wasm_min_u_i32x4:
18537 case WebAssembly::BI__builtin_wasm_max_s_i32x4:
18538 case WebAssembly::BI__builtin_wasm_max_u_i32x4: {
18542 switch (BuiltinID) {
18543 case WebAssembly::BI__builtin_wasm_min_s_i8x16:
18544 case WebAssembly::BI__builtin_wasm_min_s_i16x8:
18545 case WebAssembly::BI__builtin_wasm_min_s_i32x4:
18546 ICmp = Builder.CreateICmpSLT(LHS, RHS);
18548 case WebAssembly::BI__builtin_wasm_min_u_i8x16:
18549 case WebAssembly::BI__builtin_wasm_min_u_i16x8:
18550 case WebAssembly::BI__builtin_wasm_min_u_i32x4:
18551 ICmp = Builder.CreateICmpULT(LHS, RHS);
18553 case WebAssembly::BI__builtin_wasm_max_s_i8x16:
18554 case WebAssembly::BI__builtin_wasm_max_s_i16x8:
18555 case WebAssembly::BI__builtin_wasm_max_s_i32x4:
18556 ICmp = Builder.CreateICmpSGT(LHS, RHS);
18558 case WebAssembly::BI__builtin_wasm_max_u_i8x16:
18559 case WebAssembly::BI__builtin_wasm_max_u_i16x8:
18560 case WebAssembly::BI__builtin_wasm_max_u_i32x4:
18561 ICmp = Builder.CreateICmpUGT(LHS, RHS);
18564 llvm_unreachable(
"unexpected builtin ID");
18566 return Builder.CreateSelect(ICmp, LHS, RHS);
18568 case WebAssembly::BI__builtin_wasm_avgr_u_i8x16:
18569 case WebAssembly::BI__builtin_wasm_avgr_u_i16x8: {
18574 return Builder.CreateCall(Callee, {LHS, RHS});
18576 case WebAssembly::BI__builtin_wasm_q15mulr_sat_s_i16x8: {
18580 return Builder.CreateCall(Callee, {LHS, RHS});
18582 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
18583 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
18584 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
18585 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4: {
18588 switch (BuiltinID) {
18589 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
18590 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
18591 IntNo = Intrinsic::wasm_extadd_pairwise_signed;
18593 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
18594 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4:
18595 IntNo = Intrinsic::wasm_extadd_pairwise_unsigned;
18598 llvm_unreachable(
"unexptected builtin ID");
18602 return Builder.CreateCall(Callee, Vec);
18604 case WebAssembly::BI__builtin_wasm_bitselect: {
18610 return Builder.CreateCall(Callee, {V1, V2,
C});
18612 case WebAssembly::BI__builtin_wasm_dot_s_i32x4_i16x8: {
18616 return Builder.CreateCall(Callee, {LHS, RHS});
18618 case WebAssembly::BI__builtin_wasm_popcnt_i8x16: {
18622 return Builder.CreateCall(Callee, {Vec});
18624 case WebAssembly::BI__builtin_wasm_any_true_v128:
18625 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
18626 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
18627 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
18628 case WebAssembly::BI__builtin_wasm_all_true_i64x2: {
18630 switch (BuiltinID) {
18631 case WebAssembly::BI__builtin_wasm_any_true_v128:
18632 IntNo = Intrinsic::wasm_anytrue;
18634 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
18635 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
18636 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
18637 case WebAssembly::BI__builtin_wasm_all_true_i64x2:
18638 IntNo = Intrinsic::wasm_alltrue;
18641 llvm_unreachable(
"unexpected builtin ID");
18645 return Builder.CreateCall(Callee, {Vec});
18647 case WebAssembly::BI__builtin_wasm_bitmask_i8x16:
18648 case WebAssembly::BI__builtin_wasm_bitmask_i16x8:
18649 case WebAssembly::BI__builtin_wasm_bitmask_i32x4:
18650 case WebAssembly::BI__builtin_wasm_bitmask_i64x2: {
18654 return Builder.CreateCall(Callee, {Vec});
18656 case WebAssembly::BI__builtin_wasm_abs_f32x4:
18657 case WebAssembly::BI__builtin_wasm_abs_f64x2: {
18660 return Builder.CreateCall(Callee, {Vec});
18662 case WebAssembly::BI__builtin_wasm_sqrt_f32x4:
18663 case WebAssembly::BI__builtin_wasm_sqrt_f64x2: {
18666 return Builder.CreateCall(Callee, {Vec});
18668 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
18669 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
18670 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
18671 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4: {
18675 switch (BuiltinID) {
18676 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
18677 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
18678 IntNo = Intrinsic::wasm_narrow_signed;
18680 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
18681 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4:
18682 IntNo = Intrinsic::wasm_narrow_unsigned;
18685 llvm_unreachable(
"unexpected builtin ID");
18689 return Builder.CreateCall(Callee, {Low, High});
18691 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
18692 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4: {
18695 switch (BuiltinID) {
18696 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
18697 IntNo = Intrinsic::fptosi_sat;
18699 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4:
18700 IntNo = Intrinsic::fptoui_sat;
18703 llvm_unreachable(
"unexpected builtin ID");
18705 llvm::Type *SrcT = Vec->getType();
18706 llvm::Type *TruncT = SrcT->getWithNewType(Builder.getInt32Ty());
18708 Value *
Trunc = Builder.CreateCall(Callee, Vec);
18709 Value *Splat = Constant::getNullValue(TruncT);
18710 return Builder.CreateShuffleVector(Trunc, Splat,
ArrayRef<int>{0, 1, 2, 3});
18712 case WebAssembly::BI__builtin_wasm_shuffle_i8x16: {
18717 while (OpIdx < 18) {
18720 assert(LaneConst &&
"Constant arg isn't actually constant?");
18721 Ops[OpIdx++] = llvm::ConstantInt::get(
getLLVMContext(), *LaneConst);
18724 return Builder.CreateCall(Callee, Ops);
18726 case WebAssembly::BI__builtin_wasm_fma_f32x4:
18727 case WebAssembly::BI__builtin_wasm_fms_f32x4:
18728 case WebAssembly::BI__builtin_wasm_fma_f64x2:
18729 case WebAssembly::BI__builtin_wasm_fms_f64x2: {
18734 switch (BuiltinID) {
18735 case WebAssembly::BI__builtin_wasm_fma_f32x4:
18736 case WebAssembly::BI__builtin_wasm_fma_f64x2:
18737 IntNo = Intrinsic::wasm_fma;
18739 case WebAssembly::BI__builtin_wasm_fms_f32x4:
18740 case WebAssembly::BI__builtin_wasm_fms_f64x2:
18741 IntNo = Intrinsic::wasm_fms;
18744 llvm_unreachable(
"unexpected builtin ID");
18747 return Builder.CreateCall(Callee, {A, B,
C});
18749 case WebAssembly::BI__builtin_wasm_laneselect_i8x16:
18750 case WebAssembly::BI__builtin_wasm_laneselect_i16x8:
18751 case WebAssembly::BI__builtin_wasm_laneselect_i32x4:
18752 case WebAssembly::BI__builtin_wasm_laneselect_i64x2: {
18758 return Builder.CreateCall(Callee, {A, B,
C});
18760 case WebAssembly::BI__builtin_wasm_relaxed_swizzle_i8x16: {
18764 return Builder.CreateCall(Callee, {Src, Indices});
18766 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
18767 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
18768 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
18769 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2: {
18773 switch (BuiltinID) {
18774 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
18775 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
18776 IntNo = Intrinsic::wasm_relaxed_min;
18778 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
18779 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2:
18780 IntNo = Intrinsic::wasm_relaxed_max;
18783 llvm_unreachable(
"unexpected builtin ID");
18786 return Builder.CreateCall(Callee, {LHS, RHS});
18788 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
18789 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
18790 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
18791 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2: {
18794 switch (BuiltinID) {
18795 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
18796 IntNo = Intrinsic::wasm_relaxed_trunc_signed;
18798 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
18799 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned;
18801 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
18802 IntNo = Intrinsic::wasm_relaxed_trunc_signed_zero;
18804 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2:
18805 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned_zero;
18808 llvm_unreachable(
"unexpected builtin ID");
18811 return Builder.CreateCall(Callee, {Vec});
18813 case WebAssembly::BI__builtin_wasm_relaxed_q15mulr_s_i16x8: {
18817 return Builder.CreateCall(Callee, {LHS, RHS});
18819 case WebAssembly::BI__builtin_wasm_dot_i8x16_i7x16_s_i16x8: {
18823 return Builder.CreateCall(Callee, {LHS, RHS});
18825 case WebAssembly::BI__builtin_wasm_dot_i8x16_i7x16_add_s_i32x4: {
18831 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
18838static std::pair<Intrinsic::ID, unsigned>
18841 unsigned BuiltinID;
18842 Intrinsic::ID IntrinsicID;
18846#define CUSTOM_BUILTIN_MAPPING(x,s) \
18847 { Hexagon::BI__builtin_HEXAGON_##x, Intrinsic::hexagon_##x, s },
18879#include "clang/Basic/BuiltinsHexagonMapCustomDep.def"
18880#undef CUSTOM_BUILTIN_MAPPING
18883 auto CmpInfo = [] (Info A, Info B) {
return A.BuiltinID < B.BuiltinID; };
18884 static const bool SortOnce = (llvm::sort(Infos, CmpInfo),
true);
18887 const Info *F = std::lower_bound(std::begin(Infos), std::end(Infos),
18888 Info{BuiltinID, 0, 0}, CmpInfo);
18889 if (F == std::end(Infos) || F->BuiltinID != BuiltinID)
18890 return {Intrinsic::not_intrinsic, 0};
18892 return {F->IntrinsicID, F->VecLen};
18901 auto MakeCircOp = [
this, E](
unsigned IntID,
bool IsLoad) {
18906 llvm::Value *
Base = Builder.CreateLoad(BP);
18916 for (
unsigned i = 1, e = E->
getNumArgs(); i != e; ++i)
18922 llvm::Value *NewBase = IsLoad ? Builder.CreateExtractValue(
Result, 1)
18924 llvm::Value *LV = Builder.CreateBitCast(
18927 llvm::Value *RetVal =
18928 Builder.CreateAlignedStore(NewBase, LV, Dest.
getAlignment());
18930 RetVal = Builder.CreateExtractValue(
Result, 0);
18937 auto MakeBrevLd = [
this, E](
unsigned IntID, llvm::Type *DestTy) {
18941 llvm::Value *BaseAddress =
18950 llvm::Value *DestAddress = DestAddr.
getPointer();
18955 llvm::Value *
Result = Builder.CreateCall(
18956 CGM.
getIntrinsic(IntID), {BaseAddress, EmitScalarExpr(E->getArg(2))});
18959 llvm::Value *DestVal = Builder.CreateExtractValue(
Result, 0);
18964 DestVal = Builder.CreateTrunc(DestVal, DestTy);
18966 llvm::Value *DestForStore =
18967 Builder.CreateBitCast(DestAddress, DestVal->getType()->getPointerTo());
18968 Builder.CreateAlignedStore(DestVal, DestForStore, DestAddr.
getAlignment());
18970 return Builder.CreateExtractValue(
Result, 1);
18973 auto V2Q = [
this, VecLen] (llvm::Value *Vec) {
18974 Intrinsic::ID ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandvrt_128B
18975 : Intrinsic::hexagon_V6_vandvrt;
18977 {Vec, Builder.getInt32(-1)});
18979 auto Q2V = [
this, VecLen] (llvm::Value *Pred) {
18980 Intrinsic::ID ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandqrt_128B
18981 : Intrinsic::hexagon_V6_vandqrt;
18983 {Pred, Builder.getInt32(-1)});
18986 switch (BuiltinID) {
18990 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry:
18991 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry_128B:
18992 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry:
18993 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry_128B: {
18996 Address PredAddr = Builder.CreateElementBitCast(
18998 llvm::Value *PredIn = V2Q(Builder.CreateLoad(PredAddr));
19000 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn});
19002 llvm::Value *PredOut = Builder.CreateExtractValue(
Result, 1);
19003 Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.
getPointer(),
19005 return Builder.CreateExtractValue(
Result, 0);
19008 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq:
19009 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq:
19010 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq:
19011 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq:
19012 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq_128B:
19013 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq_128B:
19014 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq_128B:
19015 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq_128B: {
19019 if (
auto *Cast = dyn_cast<ImplicitCastExpr>(PredOp)) {
19020 if (
Cast->getCastKind() == CK_BitCast)
19021 PredOp =
Cast->getSubExpr();
19024 for (
int i = 1, e = E->
getNumArgs(); i != e; ++i)
19029 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pci:
19030 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pci:
19031 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pci:
19032 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pci:
19033 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pci:
19034 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pci:
19035 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pcr:
19036 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pcr:
19037 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pcr:
19038 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pcr:
19039 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pcr:
19040 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pcr:
19041 return MakeCircOp(ID,
true);
19042 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pci:
19043 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pci:
19044 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pci:
19045 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pci:
19046 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pci:
19047 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pcr:
19048 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pcr:
19049 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pcr:
19050 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pcr:
19051 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pcr:
19052 return MakeCircOp(ID,
false);
19053 case Hexagon::BI__builtin_brev_ldub:
19054 return MakeBrevLd(Intrinsic::hexagon_L2_loadrub_pbr,
Int8Ty);
19055 case Hexagon::BI__builtin_brev_ldb:
19056 return MakeBrevLd(Intrinsic::hexagon_L2_loadrb_pbr,
Int8Ty);
19057 case Hexagon::BI__builtin_brev_lduh:
19058 return MakeBrevLd(Intrinsic::hexagon_L2_loadruh_pbr,
Int16Ty);
19059 case Hexagon::BI__builtin_brev_ldh:
19060 return MakeBrevLd(Intrinsic::hexagon_L2_loadrh_pbr,
Int16Ty);
19061 case Hexagon::BI__builtin_brev_ldw:
19062 return MakeBrevLd(Intrinsic::hexagon_L2_loadri_pbr,
Int32Ty);
19063 case Hexagon::BI__builtin_brev_ldd:
19064 return MakeBrevLd(Intrinsic::hexagon_L2_loadrd_pbr,
Int64Ty);
19077 unsigned ICEArguments = 0;
19085 if (BuiltinID == RISCVVector::BI__builtin_rvv_vget_v ||
19086 BuiltinID == RISCVVector::BI__builtin_rvv_vset_v)
19087 ICEArguments = 1 << 1;
19092 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
19094 if ((ICEArguments & (1 << i)) == 0) {
19101 Ops.push_back(llvm::ConstantInt::get(
19105 Intrinsic::ID ID = Intrinsic::not_intrinsic;
19107 constexpr unsigned TAIL_UNDISTURBED = 0;
19111 switch (BuiltinID) {
19112 default: llvm_unreachable(
"unexpected builtin ID");
19113 case RISCV::BI__builtin_riscv_orc_b_32:
19114 case RISCV::BI__builtin_riscv_orc_b_64:
19115 case RISCV::BI__builtin_riscv_clz_32:
19116 case RISCV::BI__builtin_riscv_clz_64:
19117 case RISCV::BI__builtin_riscv_ctz_32:
19118 case RISCV::BI__builtin_riscv_ctz_64:
19119 case RISCV::BI__builtin_riscv_clmul:
19120 case RISCV::BI__builtin_riscv_clmulh:
19121 case RISCV::BI__builtin_riscv_clmulr:
19122 case RISCV::BI__builtin_riscv_bcompress_32:
19123 case RISCV::BI__builtin_riscv_bcompress_64:
19124 case RISCV::BI__builtin_riscv_bdecompress_32:
19125 case RISCV::BI__builtin_riscv_bdecompress_64:
19126 case RISCV::BI__builtin_riscv_bfp_32:
19127 case RISCV::BI__builtin_riscv_bfp_64:
19128 case RISCV::BI__builtin_riscv_grev_32:
19129 case RISCV::BI__builtin_riscv_grev_64:
19130 case RISCV::BI__builtin_riscv_gorc_32:
19131 case RISCV::BI__builtin_riscv_gorc_64:
19132 case RISCV::BI__builtin_riscv_shfl_32:
19133 case RISCV::BI__builtin_riscv_shfl_64:
19134 case RISCV::BI__builtin_riscv_unshfl_32:
19135 case RISCV::BI__builtin_riscv_unshfl_64:
19136 case RISCV::BI__builtin_riscv_xperm4:
19137 case RISCV::BI__builtin_riscv_xperm8:
19138 case RISCV::BI__builtin_riscv_xperm_n:
19139 case RISCV::BI__builtin_riscv_xperm_b:
19140 case RISCV::BI__builtin_riscv_xperm_h:
19141 case RISCV::BI__builtin_riscv_xperm_w:
19142 case RISCV::BI__builtin_riscv_crc32_b:
19143 case RISCV::BI__builtin_riscv_crc32_h:
19144 case RISCV::BI__builtin_riscv_crc32_w:
19145 case RISCV::BI__builtin_riscv_crc32_d:
19146 case RISCV::BI__builtin_riscv_crc32c_b:
19147 case RISCV::BI__builtin_riscv_crc32c_h:
19148 case RISCV::BI__builtin_riscv_crc32c_w:
19149 case RISCV::BI__builtin_riscv_crc32c_d:
19150 case RISCV::BI__builtin_riscv_fsl_32:
19151 case RISCV::BI__builtin_riscv_fsr_32:
19152 case RISCV::BI__builtin_riscv_fsl_64:
19153 case RISCV::BI__builtin_riscv_fsr_64:
19154 case RISCV::BI__builtin_riscv_brev8:
19155 case RISCV::BI__builtin_riscv_zip_32:
19156 case RISCV::BI__builtin_riscv_unzip_32: {
19157 switch (BuiltinID) {
19158 default: llvm_unreachable(
"unexpected builtin ID");
19160 case RISCV::BI__builtin_riscv_orc_b_32:
19161 case RISCV::BI__builtin_riscv_orc_b_64:
19162 ID = Intrinsic::riscv_orc_b;
19164 case RISCV::BI__builtin_riscv_clz_32:
19165 case RISCV::BI__builtin_riscv_clz_64: {
19167 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(
false)});
19169 case RISCV::BI__builtin_riscv_ctz_32:
19170 case RISCV::BI__builtin_riscv_ctz_64: {
19172 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(
false)});
19176 case RISCV::BI__builtin_riscv_clmul:
19177 ID = Intrinsic::riscv_clmul;
19179 case RISCV::BI__builtin_riscv_clmulh:
19180 ID = Intrinsic::riscv_clmulh;
19182 case RISCV::BI__builtin_riscv_clmulr:
19183 ID = Intrinsic::riscv_clmulr;
19187 case RISCV::BI__builtin_riscv_bcompress_32:
19188 case RISCV::BI__builtin_riscv_bcompress_64:
19189 ID = Intrinsic::riscv_bcompress;
19191 case RISCV::BI__builtin_riscv_bdecompress_32:
19192 case RISCV::BI__builtin_riscv_bdecompress_64:
19193 ID = Intrinsic::riscv_bdecompress;
19197 case RISCV::BI__builtin_riscv_bfp_32:
19198 case RISCV::BI__builtin_riscv_bfp_64:
19199 ID = Intrinsic::riscv_bfp;
19203 case RISCV::BI__builtin_riscv_grev_32:
19204 case RISCV::BI__builtin_riscv_grev_64:
19205 ID = Intrinsic::riscv_grev;
19207 case RISCV::BI__builtin_riscv_gorc_32:
19208 case RISCV::BI__builtin_riscv_gorc_64:
19209 ID = Intrinsic::riscv_gorc;
19211 case RISCV::BI__builtin_riscv_shfl_32:
19212 case RISCV::BI__builtin_riscv_shfl_64:
19213 ID = Intrinsic::riscv_shfl;
19215 case RISCV::BI__builtin_riscv_unshfl_32:
19216 case RISCV::BI__builtin_riscv_unshfl_64:
19217 ID = Intrinsic::riscv_unshfl;
19219 case RISCV::BI__builtin_riscv_xperm_n:
19220 ID = Intrinsic::riscv_xperm_n;
19222 case RISCV::BI__builtin_riscv_xperm_b:
19223 ID = Intrinsic::riscv_xperm_b;
19225 case RISCV::BI__builtin_riscv_xperm_h:
19226 ID = Intrinsic::riscv_xperm_h;
19228 case RISCV::BI__builtin_riscv_xperm_w:
19229 ID = Intrinsic::riscv_xperm_w;
19233 case RISCV::BI__builtin_riscv_crc32_b:
19234 ID = Intrinsic::riscv_crc32_b;
19236 case RISCV::BI__builtin_riscv_crc32_h:
19237 ID = Intrinsic::riscv_crc32_h;
19239 case RISCV::BI__builtin_riscv_crc32_w:
19240 ID = Intrinsic::riscv_crc32_w;
19242 case RISCV::BI__builtin_riscv_crc32_d:
19243 ID = Intrinsic::riscv_crc32_d;
19245 case RISCV::BI__builtin_riscv_crc32c_b:
19246 ID = Intrinsic::riscv_crc32c_b;
19248 case RISCV::BI__builtin_riscv_crc32c_h:
19249 ID = Intrinsic::riscv_crc32c_h;
19251 case RISCV::BI__builtin_riscv_crc32c_w:
19252 ID = Intrinsic::riscv_crc32c_w;
19254 case RISCV::BI__builtin_riscv_crc32c_d:
19255 ID = Intrinsic::riscv_crc32c_d;
19259 case RISCV::BI__builtin_riscv_fsl_32:
19260 case RISCV::BI__builtin_riscv_fsl_64:
19261 ID = Intrinsic::riscv_fsl;
19263 case RISCV::BI__builtin_riscv_fsr_32:
19264 case RISCV::BI__builtin_riscv_fsr_64:
19265 ID = Intrinsic::riscv_fsr;
19269 case RISCV::BI__builtin_riscv_xperm8:
19270 ID = Intrinsic::riscv_xperm8;
19272 case RISCV::BI__builtin_riscv_xperm4:
19273 ID = Intrinsic::riscv_xperm4;
19277 case RISCV::BI__builtin_riscv_brev8:
19278 ID = Intrinsic::riscv_brev8;
19280 case RISCV::BI__builtin_riscv_zip_32:
19281 ID = Intrinsic::riscv_zip;
19283 case RISCV::BI__builtin_riscv_unzip_32:
19284 ID = Intrinsic::riscv_unzip;
19288 IntrinsicTypes = {ResultType};
19295 case RISCV::BI__builtin_riscv_aes32dsi_32:
19296 ID = Intrinsic::riscv_aes32dsi;
19298 case RISCV::BI__builtin_riscv_aes32dsmi_32:
19299 ID = Intrinsic::riscv_aes32dsmi;
19301 case RISCV::BI__builtin_riscv_aes64ds_64:
19302 ID = Intrinsic::riscv_aes64ds;
19304 case RISCV::BI__builtin_riscv_aes64dsm_64:
19305 ID = Intrinsic::riscv_aes64dsm;
19307 case RISCV::BI__builtin_riscv_aes64im_64:
19308 ID = Intrinsic::riscv_aes64im;
19312 case RISCV::BI__builtin_riscv_aes32esi_32:
19313 ID = Intrinsic::riscv_aes32esi;
19315 case RISCV::BI__builtin_riscv_aes32esmi_32:
19316 ID = Intrinsic::riscv_aes32esmi;
19318 case RISCV::BI__builtin_riscv_aes64es_64:
19319 ID = Intrinsic::riscv_aes64es;
19321 case RISCV::BI__builtin_riscv_aes64esm_64:
19322 ID = Intrinsic::riscv_aes64esm;
19326 case RISCV::BI__builtin_riscv_aes64ks1i_64:
19327 ID = Intrinsic::riscv_aes64ks1i;
19329 case RISCV::BI__builtin_riscv_aes64ks2_64:
19330 ID = Intrinsic::riscv_aes64ks2;
19334 case RISCV::BI__builtin_riscv_sha256sig0:
19335 ID = Intrinsic::riscv_sha256sig0;
19336 IntrinsicTypes = {ResultType};
19338 case RISCV::BI__builtin_riscv_sha256sig1:
19339 ID = Intrinsic::riscv_sha256sig1;
19340 IntrinsicTypes = {ResultType};
19342 case RISCV::BI__builtin_riscv_sha256sum0:
19343 ID = Intrinsic::riscv_sha256sum0;
19344 IntrinsicTypes = {ResultType};
19346 case RISCV::BI__builtin_riscv_sha256sum1:
19347 ID = Intrinsic::riscv_sha256sum1;
19348 IntrinsicTypes = {ResultType};
19350 case RISCV::BI__builtin_riscv_sha512sig0_64:
19351 ID = Intrinsic::riscv_sha512sig0;
19353 case RISCV::BI__builtin_riscv_sha512sig0h_32:
19354 ID = Intrinsic::riscv_sha512sig0h;
19356 case RISCV::BI__builtin_riscv_sha512sig0l_32:
19357 ID = Intrinsic::riscv_sha512sig0l;
19359 case RISCV::BI__builtin_riscv_sha512sig1_64:
19360 ID = Intrinsic::riscv_sha512sig1;
19362 case RISCV::BI__builtin_riscv_sha512sig1h_32:
19363 ID = Intrinsic::riscv_sha512sig1h;
19365 case RISCV::BI__builtin_riscv_sha512sig1l_32:
19366 ID = Intrinsic::riscv_sha512sig1l;
19368 case RISCV::BI__builtin_riscv_sha512sum0_64:
19369 ID = Intrinsic::riscv_sha512sum0;
19371 case RISCV::BI__builtin_riscv_sha512sum0r_32:
19372 ID = Intrinsic::riscv_sha512sum0r;
19374 case RISCV::BI__builtin_riscv_sha512sum1_64:
19375 ID = Intrinsic::riscv_sha512sum1;
19377 case RISCV::BI__builtin_riscv_sha512sum1r_32:
19378 ID = Intrinsic::riscv_sha512sum1r;
19382 case RISCV::BI__builtin_riscv_sm4ks:
19383 ID = Intrinsic::riscv_sm4ks;
19384 IntrinsicTypes = {ResultType};
19386 case RISCV::BI__builtin_riscv_sm4ed:
19387 ID = Intrinsic::riscv_sm4ed;
19388 IntrinsicTypes = {ResultType};
19392 case RISCV::BI__builtin_riscv_sm3p0:
19393 ID = Intrinsic::riscv_sm3p0;
19394 IntrinsicTypes = {ResultType};
19396 case RISCV::BI__builtin_riscv_sm3p1:
19397 ID = Intrinsic::riscv_sm3p1;
19398 IntrinsicTypes = {ResultType};
19402#include "clang/Basic/riscv_vector_builtin_cg.inc"
19405 assert(ID != Intrinsic::not_intrinsic);
19408 return Builder.CreateCall(F, Ops,
"");
Defines the clang::ASTContext interface.
static SVal getValue(SVal val, SValBuilder &svalBuilder)
#define X86_CPU_SUBTYPE(ENUM, STR)
#define X86_VENDOR(ENUM, STRING)
#define X86_CPU_TYPE_ALIAS(ENUM, ALIAS)
#define X86_CPU_TYPE(ENUM, STR)
static const Builtin::Info BuiltinInfo[]
llvm::Expected< T > Expected
static void Accumulate(SMap &SM, CFGBlock *B)
static Value * EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, SpecialRegisterAccessKind AccessKind, StringRef SysReg="")
static llvm::Value * ARMMVEVectorReinterpret(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *DestType)
static Value * MakeBinaryAtomicValue(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
Utility to insert an atomic instruction based on Intrinsic::ID and the expression node.
static char bitActionToX86BTCode(BitTest::ActionKind A)
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
static Value * EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering)
#define INTRINSIC_X86_XSAVE_ID(NAME)
static CanQualType getOSLogArgType(ASTContext &C, int Size)
Get the argument type for arguments to os_log_helper.
static llvm::VectorType * GetFloatNeonType(CodeGenFunction *CGF, NeonTypeFlags IntTypeFlags)
static llvm::Value * MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V, uint32_t Shift, bool Unsigned)
static bool areBOSTypesCompatible(int From, int To)
Checks if using the result of __builtin_object_size(p, From) in place of __builtin_object_size(p,...
static llvm::Value * SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V, llvm::Type *T, bool Unsigned)
static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static Value * EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< Value * > &Ops, llvm::Triple::ArchType Arch)
#define MMA_VARIANTS_B1_AND(geom, type)
static bool AArch64SISDIntrinsicsProvenSorted
static Value * EmitX86CompressExpand(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsCompress)
static const ARMVectorIntrinsicInfo AArch64SVEIntrinsicMap[]
static bool HasExtraNeonArgument(unsigned BuiltinID)
Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the ve...
static bool TypeRequiresBuiltinLaunderImp(const ASTContext &Ctx, QualType Ty, llvm::SmallPtrSetImpl< const Decl * > &Seen)
static Value * EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static std::pair< Intrinsic::ID, unsigned > getIntrinsicForHexagonNonClangBuiltin(unsigned BuiltinID)
static Value * emitRangedBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, int low, int high)
#define MMA_INTR(geom_op_type, layout)
static Value * EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC, bool Signed, ArrayRef< Value * > Ops)
static Value * emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
static bool AArch64SVEIntrinsicsProvenSorted
static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind, const CallExpr *E)
MSVC handles setjmp a bit differently on different platforms.
static const ARMVectorIntrinsicInfo * findARMVectorIntrinsicInMap(ArrayRef< ARMVectorIntrinsicInfo > IntrinsicMap, unsigned BuiltinID, bool &MapProvenSorted)
static Value * EmitScalarFMAExpr(CodeGenFunction &CGF, const CallExpr *E, MutableArrayRef< Value * > Ops, Value *Upper, bool ZeroMask=false, unsigned PTIdx=0, bool NegAcc=false)
#define MUTATE_LDBL(func)
static Value * EmitX86ExpandLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
static Value * emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static Value * EmitX86MaskedLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
static bool TypeRequiresBuiltinLaunder(CodeGenModule &CGM, QualType Ty)
Determine if the specified type requires laundering by checking if it is a dynamic class type or cont...
static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static Value * EmitISOVolatileLoad(CodeGenFunction &CGF, const CallExpr *E)
static struct WidthAndSignedness EncompassingIntegerType(ArrayRef< struct WidthAndSignedness > Types)
static Integer GetIntegerConstantValue(const Expr *E, ASTContext &Context)
#define MMA_VARIANTS(geom, type)
static llvm::Value * VectorZip(CGBuilderTy &Builder, llvm::Value *V0, llvm::Value *V1)
static Value * EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
constexpr unsigned SVEBitsPerBlock
#define NEONMAP0(NameBase)
static Value * EmitX86MaskedStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
static Value * emitBinaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static Value * emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, Instruction::BinaryOps Op, bool Invert=false)
Utility to insert an atomic instruction based Intrinsic::ID and the expression node,...
static Value * EmitX86Muldq(CodeGenFunction &CGF, bool IsSigned, ArrayRef< Value * > Ops)
static llvm::Value * ARMMVEVectorElementReverse(CGBuilderTy &Builder, llvm::Value *V, unsigned ReverseWidth)
static Optional< CodeGenFunction::MSVCIntrin > translateAarch64ToMsvcIntrin(unsigned BuiltinID)
#define MMA_SATF_VARIANTS(geom, type)
static int64_t clamp(int64_t Value, int64_t Low, int64_t High)
static llvm::Value * EmitBitTestIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
Emit a _bittest* intrinsic.
static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap[]
static Value * EmitSignBit(CodeGenFunction &CGF, Value *V)
Emit the computation of the sign bit for a floating point value.
static Value * EmitFAbs(CodeGenFunction &CGF, Value *V)
EmitFAbs - Emit a call to @llvm.fabs().
#define CUSTOM_BUILTIN_MAPPING(x, s)
static Value * EmitX86CvtF16ToFloatExpr(CodeGenFunction &CGF, ArrayRef< Value * > Ops, llvm::Type *DstTy)
static bool isSpecialUnsignedMultiplySignedResult(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
static llvm::Value * getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType)
static llvm::Value * emitPPCLoadReserveIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
static llvm::Value * VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd)
static Optional< CodeGenFunction::MSVCIntrin > translateX86ToMsvcIntrin(unsigned BuiltinID)
static Value * EmitX86SExtMask(CodeGenFunction &CGF, Value *Op, llvm::Type *DstTy)
static Value * emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static WidthAndSignedness getIntegerWidthAndSignedness(const clang::ASTContext &context, const clang::QualType Type)
static Value * EmitX86FunnelShift(CodeGenFunction &CGF, Value *Op0, Value *Op1, Value *Amt, bool IsRight)
static RValue EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
Emit a checked mixed-sign multiply.
static llvm::ScalableVectorType * getSVEVectorForElementType(llvm::Type *EltTy)
static unsigned mutateLongDoubleBuiltin(unsigned BuiltinID)
#define INTRINSIC_WITH_CC(NAME)
static llvm::FixedVectorType * GetNeonType(CodeGenFunction *CGF, NeonTypeFlags TypeFlags, bool HasLegalHalfType=true, bool V1Ty=false, bool AllowBFloatArgsAndRet=true)
static RValue EmitBinaryAtomic(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E)
static llvm::Value * ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT)
static Value * EmitX86MaskLogic(CodeGenFunction &CGF, Instruction::BinaryOps Opc, ArrayRef< Value * > Ops, bool InvertLHS=false)
static Value * EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::Type *ResultType)
static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size, Align AlignmentInBytes)
static Value * EmitX86Select(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
SpecialRegisterAccessKind
static Value * EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering=AtomicOrdering::SequentiallyConsistent)
This function should be invoked to emit atomic cmpxchg for Microsoft's _InterlockedCompareExchange* i...
static Value * EmitX86FMAExpr(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, unsigned BuiltinID, bool IsAddSub)
static Value * getMaskVecValue(CodeGenFunction &CGF, Value *Mask, unsigned NumElts)
static bool isSpecialMixedSignMultiply(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
Determine if a binop is a checked mixed-sign multiply we can specialize.
static Value * MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, bool ReturnBool)
Utility to insert an atomic cmpxchg instruction.
static Optional< CodeGenFunction::MSVCIntrin > translateArmToMsvcIntrin(unsigned BuiltinID)
static Value * EmitToInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::IntegerType *IntType)
Emit the conversions required to turn the given value into an integer of the given size.
static llvm::Value * ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V)
static Value * EmitX86MaskedCompareResult(CodeGenFunction &CGF, Value *Cmp, unsigned NumElts, Value *MaskIn)
static Value * EmitX86CompressStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
static bool NEONSIMDIntrinsicsProvenSorted
static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[]
static Value * EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E)
static llvm::Value * EmitOverflowIntrinsic(CodeGenFunction &CGF, const llvm::Intrinsic::ID IntrinsicID, llvm::Value *X, llvm::Value *Y, llvm::Value *&Carry)
Emit a call to llvm.
static Value * EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, const ARMVectorIntrinsicInfo &SISDInfo, SmallVectorImpl< Value * > &Ops, const CallExpr *E)
static Value * emitFPIntBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static Value * emitTernaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
#define MMA_LDST(n, geom_op_type)
static Value * EmitX86vpcom(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsSigned)
@ _InterlockedExchangeAdd_rel
@ _InterlockedIncrement_acq
@ _InterlockedExchange_nf
@ _InterlockedIncrement_nf
@ _InterlockedExchange_acq
@ _InterlockedCompareExchange128_rel
@ _InterlockedCompareExchange128_acq
@ _InterlockedCompareExchange_acq
@ _InterlockedExchangeAdd_nf
@ _InterlockedCompareExchange_nf
@ _InterlockedDecrement_rel
@ _InterlockedExchangeSub
@ _InterlockedExchangeAdd_acq
@ _InterlockedIncrement_rel
@ _InterlockedCompareExchange128_nf
@ _InterlockedCompareExchange128
@ _InterlockedExchange_rel
@ _InterlockedDecrement_nf
@ _InterlockedExchangeAdd
@ _InterlockedDecrement_acq
@ _InterlockedCompareExchange_rel
static Value * EmitX86ConvertToMask(CodeGenFunction &CGF, Value *In)
static Value * EmitISOVolatileStore(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, unsigned IntrinsicID, const CallExpr *E)
Handle a SystemZ function in which the final argument is a pointer to an int that receives the post-i...
static llvm::Value * EmitX86BitTestIntrinsic(CodeGenFunction &CGF, BitTest BT, const CallExpr *E, Value *BitBase, Value *BitPos)
static RValue EmitCheckedUnsignedMultiplySignedResult(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
static Value * emitUnaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, llvm::StringRef Name="")
static Value * EmitX86CvtBF16ToFloatExpr(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops)
static Value * emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID, llvm::Type *Ty, ArrayRef< Value * > Args)
static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD, const CallExpr *E, llvm::Constant *calleeValue)
static Value * EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitX86Ternlog(CodeGenFunction &CGF, bool ZeroMask, ArrayRef< Value * > Ops)
static llvm::AtomicOrdering getBitTestAtomicOrdering(BitTest::InterlockingKind I)
#define MMA_VARIANTS_B1_XOR(geom, type)
#define MMA_VARIANTS_I4(geom, type)
static Value * EmitX86ConvertIntToFp(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, bool IsSigned)
static Value * packTBLDVectorList(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Value *ExtOp, Value *IndexOp, llvm::Type *ResTy, unsigned IntID, const char *Name)
static Value * EmitX86ScalarSelect(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
static Value * EmitAtomicDecrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static bool AArch64SIMDIntrinsicsProvenSorted
static const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[]
CodeGenFunction::ComplexPairTy ComplexPairTy
static CompilationDatabasePluginRegistry::Add< FixedCompilationDatabasePlugin > X("fixed-compilation-database", "Reads plain-text flags file")
#define ALIAS(NAME, TOK, FLAGS)
static bool Ret(InterpState &S, CodePtr &PC, APValue &Result)
static std::string toString(const clang::SanitizerSet &Sanitizers)
Produce a string containing comma-separated names of sanitizers in Sanitizers set.
Enumerates target-specific builtins in their own namespaces within namespace clang.
__DEVICE__ float modf(float __x, float *__iptr)
__DEVICE__ double nan(const char *)
__device__ __2f16 float bool s
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
unsigned getIntWidth(QualType T) const
Builtin::Context & BuiltinInfo
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
TypeInfo getTypeInfo(const Type *T) const
Get the size and alignment of the specified complete type in bits.
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArrayType::ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
QualType getObjCIdType() const
Represents the Objective-CC id type.
bool hasSameUnqualifiedType(QualType T1, QualType T2) const
Determine whether the given types are equivalent after cvr-qualifiers have been removed.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error, unsigned *IntegerConstantArgs=nullptr) const
Return the type for the specified builtin.
const TargetInfo & getTargetInfo() const
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
@ GE_Missing_type
Missing a type.
unsigned getTargetAddressSpace(QualType T) const
bool isLibFunction(unsigned ID) const
Return true if this is a builtin for a libc/libm function, with a "__builtin_" prefix (e....
const char * getName(unsigned ID) const
Return the identifier name for the specified builtin, e.g.
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Expr * getArg(unsigned Arg)
getArg - Return the specified argument.
FunctionDecl * getDirectCallee()
If the callee is a FunctionDecl, return it. Otherwise return null.
unsigned getNumArgs() const
getNumArgs - Return the number of actual arguments to this call.
QualType getCallReturnType(const ASTContext &Ctx) const
getCallReturnType - Get the return type of the call expr.
CharUnits - This is an opaque type for sizes expressed in character units.
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
static CharUnits One()
One - Construct a CharUnits quantity of one.
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
XRayInstrSet XRayInstrumentationBundle
Set of XRay instrumentation kinds to emit.
virtual bool allowBFloatArgsAndRet() const
CharUnits getAlignment() const
Return the alignment of this pointer.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
llvm::Value * getPointer() const
llvm::PointerType * getType() const
Return the type of the pointer value.
Address getAddress() const
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
llvm::StoreInst * CreateAlignedStore(llvm::Value *Val, llvm::Value *Addr, CharUnits Align, bool IsVolatile=false)
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, llvm::Value *Ptr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
llvm::LoadInst * CreateAlignedLoad(llvm::Type *Ty, llvm::Value *Addr, CharUnits Align, const llvm::Twine &Name="")
Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
llvm::AtomicCmpXchgInst * CreateAtomicCmpXchg(llvm::Value *Ptr, llvm::Value *Cmp, llvm::Value *New, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
Address CreateGEP(Address Addr, llvm::Value *Index, const llvm::Twine &Name="")
virtual std::string getDeviceSideName(const NamedDecl *ND)=0
Returns function or variable name on device side even if the current compilation is for host.
virtual llvm::GlobalVariable * getThrowInfo(QualType T)
All available information about a concrete callee.
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
llvm::DIType * getOrCreateStandaloneType(QualType Ty, SourceLocation Loc)
Emit standalone debug info for a type.
CGFunctionInfo - Class to encapsulate the information about a function definition.
virtual void EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr, llvm::Value *Size)=0
EnqueuedBlockInfo emitOpenCLEnqueuedBlock(CodeGenFunction &CGF, const Expr *E)
CallArgList - Type for representing both the value and type of arguments in a call.
void add(RValue rvalue, QualType type)
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::Value * EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
llvm::Value * EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
llvm::Value * EmitLifetimeStart(llvm::TypeSize Size, llvm::Value *Addr)
std::pair< RValue, llvm::Value * > EmitAtomicCompareExchange(LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success=llvm::AtomicOrdering::SequentiallyConsistent, llvm::AtomicOrdering Failure=llvm::AtomicOrdering::SequentiallyConsistent, bool IsWeak=false, AggValueSlot Slot=AggValueSlot::ignored())
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
llvm::Value * EmitSVEPredicateCast(llvm::Value *Pred, llvm::ScalableVectorType *VTy)
llvm::CallInst * EmitTrapCall(llvm::Intrinsic::ID IntrID)
Emit a call to trap or debugtrap and attach function attribute "trap-func-name" if specified.
SanitizerSet SanOpts
Sanitizers enabled for this function.
RValue EmitBuiltinIsAligned(const CallExpr *E)
Emit IR for __builtin_is_aligned.
LValue EmitAggExprToLValue(const Expr *E)
EmitAggExprToLValue - Emit the computation of the specified expression of aggregate type into a tempo...
void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc, AbstractCallee AC, unsigned ParmNum)
Create a check for a function parameter that may potentially be declared as non-null.
llvm::Value * EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr)
void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
llvm::Value * EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind)
Emits an argument for a call to a builtin.
llvm::Value * EmitSVEGatherLoad(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
CleanupKind getARCCleanupKind()
Retrieves the default cleanup kind for an ARC cleanup.
llvm::Value * EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
RValue emitBuiltinOSLogFormat(const CallExpr &E)
Emit IR for __builtin_os_log_format.
llvm::Value * EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart)
Emits a call to an LLVM variable-argument intrinsic, either llvm.va_start or llvm....
llvm::Value * EmitSVEMaskedStore(const CallExpr *, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
llvm::Value * EmitSEHExceptionInfo()
RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp)
Emit IR for __builtin_align_up/__builtin_align_down.
const LangOptions & getLangOpts() const
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitUnreachable(SourceLocation Loc)
Emit a reached-unreachable diagnostic if Loc is valid and runtime checking is enabled.
ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal=false, bool IgnoreImag=false)
EmitComplexExpr - Emit the computation of the specified expression of complex type,...
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **callOrInvoke, bool IsMustTail, SourceLocation Loc)
EmitCall - Generate a call of the given function, expecting the given result type,...
llvm::Value * EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx, const llvm::ElementCount &Count)
llvm::Type * ConvertTypeForMem(QualType T)
llvm::Value * EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID, bool IsZExtReturn)
bool AlwaysEmitXRayCustomEvents() const
AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit XRay custom event handling c...
llvm::Value * EmitSVEDupX(llvm::Value *Scalar)
@ Default
! No language constraints on evaluation order.
const TargetInfo & getTarget() const
llvm::Value * vectorWrapScalar16(llvm::Value *Op)
llvm::Function * LookupNeonLLVMIntrinsic(unsigned IntrinsicID, unsigned Modifier, llvm::Type *ArgTy, const CallExpr *E)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
llvm::Value * EmitSEHExceptionCode()
llvm::Value * EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
EmitTargetBuiltinExpr - Emit the given builtin call.
void pushCleanupAfterFullExpr(CleanupKind Kind, As... A)
Queue a cleanup to be pushed after finishing the current full-expression, potentially with an active ...
RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID)
RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E)
Address EmitArrayToPointerDecay(const Expr *Array, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
void EmitCheck(ArrayRef< std::pair< llvm::Value *, SanitizerMask > > Checked, SanitizerHandler Check, ArrayRef< llvm::Constant * > StaticArgs, ArrayRef< llvm::Value * > DynamicArgs)
Create a basic block that will either trap or call a handler function in the UBSan runtime with the p...
RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type, const CallExpr *TheCallExpr, bool IsDelete)
RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty, const llvm::CmpInst::Predicate Fp, const llvm::CmpInst::Predicate Ip, const llvm::Twine &Name="")
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
llvm::ScalableVectorType * getSVEType(const SVETypeFlags &TypeFlags)
RValue emitRotate(const CallExpr *E, bool IsRotateRight)
llvm::Constant * EmitCheckSourceLocation(SourceLocation Loc)
Emit a description of a source location in a format suitable for passing to a runtime sanitizer handl...
void ErrorUnsupported(const Stmt *S, const char *Type)
ErrorUnsupported - Print out an error that codegen doesn't support the specified stmt yet.
CGDebugInfo * getDebugInfo()
Address EmitVAListRef(const Expr *E)
llvm::Value * EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty, bool negateForRightShift)
llvm::Value * EmitSVEMovl(const SVETypeFlags &TypeFlags, llvm::ArrayRef< llvm::Value * > Ops, unsigned BuiltinID)
void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
const TargetCodeGenInfo & getTargetHooks() const
llvm::Value * EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::Type * getEltType(const SVETypeFlags &TypeFlags)
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
bool ShouldXRayInstrumentFunction() const
ShouldXRayInstrument - Return true if the current function should be instrumented with XRay nop sleds...
llvm::Value * EmitSVEPMull(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation.
RValue EmitOpenMPDevicePrintfCallExpr(const CallExpr *E)
bool IsInPreservedAIRegion
True if CodeGen currently emits code inside presereved access index region.
llvm::Value * EmitARCRetain(QualType type, llvm::Value *value)
bool AlwaysEmitXRayTypedEvents() const
AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit XRay typed event handling ...
llvm::Value * EmitSVEScatterStore(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
ASTContext & getContext() const
llvm::Function * generateBuiltinOSLogHelperFunction(const analyze_os_log::OSLogBufferLayout &Layout, CharUnits BufferAlignment)
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
CGCallee EmitCallee(const Expr *E)
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl)
llvm::Value * BuildVector(ArrayRef< llvm::Value * > Ops)
llvm::Value * EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Type * ConvertType(QualType T)
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
llvm::CallBase * EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee, ArrayRef< llvm::Value * > args, const Twine &name="")
llvm::Value * EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
llvm::Value * EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
static Destroyer destroyARCStrongPrecise
llvm::Value * EmitCastToVoidPtr(llvm::Value *value)
Emit a cast to void* in the appropriate address space.
void EmitARCIntrinsicUse(ArrayRef< llvm::Value * > values)
RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E)
llvm::Value * EmitSVEStructLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Address EmitMSVAListRef(const Expr *E)
Emit a "reference" to a __builtin_ms_va_list; this is always the value of the expression,...
llvm::Value * EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr, ArrayRef< llvm::Value * > IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
llvm::Value * EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt, llvm::Type *Ty, bool usgn, const char *name)
bool ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope, llvm::AtomicOrdering &AO, llvm::SyncScope::ID &SSID)
SmallVector< llvm::Type *, 2 > getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
llvm::Value * EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::Value * EmitSVEStructStore(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
llvm::Value * EmitSEHAbnormalTermination()
llvm::Value * EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E)
AggValueSlot CreateAggTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateAggTemp - Create a temporary memory object for the given aggregate type.
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
llvm::Value * EmitSVEAllTruePred(const SVETypeFlags &TypeFlags)
RValue GetUndefRValue(QualType Ty)
GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
LValue EmitLValue(const Expr *E)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
llvm::Type * SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags)
SVEBuiltinMemEltTy - Returns the memory element type for this memory access builtin.
llvm::LLVMContext & getLLVMContext()
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
llvm::Value * EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, const char *NameHint, unsigned Modifier, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, Address PtrOp0, Address PtrOp1, llvm::Triple::ArchType Arch)
llvm::Value * EmitNeonCall(llvm::Function *F, SmallVectorImpl< llvm::Value * > &O, const char *name, unsigned shift=0, bool rightshift=false)
llvm::Value * EmitAnnotationCall(llvm::Function *AnnotationFn, llvm::Value *AnnotatedVal, StringRef AnnotationStr, SourceLocation Location, const AnnotateAttr *Attr)
Emit an annotation call (intrinsic).
llvm::ScalableVectorType * getSVEPredType(const SVETypeFlags &TypeFlags)
llvm::Value * EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
llvm::Value * EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
This class organizes the cross-function state that is used while generating LLVM code.
llvm::Module & getModule() const
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name.
llvm::Constant * getBuiltinLibFunction(const FunctionDecl *FD, unsigned BuiltinID)
Given a builtin id for a function like "__builtin_fabsf", return a Function* for "fabsf".
void ErrorUnsupported(const Stmt *S, const char *Type)
Print out an error that codegen doesn't support the specified stmt yet.
CGCUDARuntime & getCUDARuntime()
Return a reference to the configured CUDA runtime.
CodeGenTypes & getTypes()
CGOpenCLRuntime & getOpenCLRuntime()
Return a reference to the configured OpenCL runtime.
const TargetInfo & getTarget() const
const llvm::DataLayout & getDataLayout() const
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys=None)
void Error(SourceLocation loc, StringRef error)
Emit a general error that something can't be done.
CGCXXABI & getCXXABI() const
llvm::Constant * GetFunctionStart(const ValueDecl *Decl)
const llvm::Triple & getTriple() const
void DecorateInstructionWithTBAA(llvm::Instruction *Inst, TBAAAccessInfo TBAAInfo)
DecorateInstructionWithTBAA - Decorate the instruction with a TBAA tag.
llvm::Constant * CreateRuntimeVariable(llvm::Type *Ty, StringRef Name)
Create a new runtime global variable with the specified type and name.
TBAAAccessInfo getTBAAAccessInfo(QualType AccessType)
getTBAAAccessInfo - Get TBAA information that describes an access to an object of the given type.
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
CharUnits getNaturalPointeeTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
llvm::LLVMContext & getLLVMContext()
CGObjCRuntime & getObjCRuntime()
Return a reference to the configured Objective-C runtime.
void SetLLVMFunctionAttributes(GlobalDecl GD, const CGFunctionInfo &Info, llvm::Function *F, bool IsThunk)
Set the LLVM function attributes (sext, zext, etc).
void SetLLVMFunctionAttributesForDefinition(const Decl *D, llvm::Function *F)
Set the LLVM function attributes which only apply to a function definition.
ConstantAddress GetAddrOfConstantCString(const std::string &Str, const char *GlobalName=nullptr)
Returns a pointer to a character array containing the literal and a terminating '\0' character.
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
const CGFunctionInfo & arrangeBuiltinFunctionCall(QualType resultType, const CallArgList &args)
llvm::Constant * emitAbstract(const Expr *E, QualType T)
Emit the result of the given expression as an abstract constant, asserting that it succeeded.
Information for lazily generating a cleanup.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
llvm::Value * getBitFieldPointer() const
llvm::Value * getPointer(CodeGenFunction &CGF) const
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue getIgnored()
static RValue get(llvm::Value *V)
static RValue getAggregate(Address addr, bool isVolatile=false)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
virtual llvm::Value * encodeReturnAddress(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const
Performs the code-generation required to convert the address of an instruction into a return address ...
virtual llvm::Value * decodeReturnAddress(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const
Performs the code-generation required to convert a return address as stored by the system into the ac...
const ABIInfo & getABIInfo() const
getABIInfo() - Returns ABI info helper for the target.
virtual int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const
Determines the DWARF register number for the stack pointer, for exception-handling purposes.
Complex values, per C99 6.2.5p11.
Represents a concrete matrix type with constant number of rows and columns.
FunctionDecl * getAsFunction() LLVM_READONLY
Returns the function itself, or the templated function if this is a function template.
This represents one expression.
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
bool EvaluateAsFloat(llvm::APFloat &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsFloat - Return true if this is a constant which we can fold and convert to a floating point...
Expr * IgnoreParens() LLVM_READONLY
Skip past any parentheses which might surround this expression until reaching a fixed point.
@ NPC_ValueDependentIsNotNull
Specifies that a value-dependent expression should be considered to never be a null pointer constant.
ExprObjectKind getObjectKind() const
getObjectKind - The object kind that this expression produces.
bool EvaluateAsRValue(EvalResult &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsRValue - Return true if this is a constant which we can fold to an rvalue using any crazy t...
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point.
NullPointerConstantKind isNullPointerConstant(ASTContext &Ctx, NullPointerConstantValueDependence NPC) const
isNullPointerConstant - C99 6.3.2.3p3 - Test if this reduces down to a Null pointer constant.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx, SourceLocation *Loc=nullptr, bool isEvaluated=true) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
bool tryEvaluateObjectSize(uint64_t &Result, ASTContext &Ctx, unsigned Type) const
If the current Expr is a pointer, this will try to statically determine the number of bytes available...
const ValueDecl * getAsBuiltinConstantDeclRef(const ASTContext &Context) const
If this expression is an unambiguous reference to a single declaration, in the style of __builtin_fun...
Represents a member of a struct/union/class.
Represents a function declaration or definition.
const ParmVarDecl * getParamDecl(unsigned i) const
Represents a prototype with parameter type info, e.g.
GlobalDecl - represents a global declaration.
const Decl * getDecl() const
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
@ Other
Other implicit parameter.
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
Flags to identify the types for overloaded Neon builtins.
EltType getEltType() const
PointerType - C99 6.7.5.1 - Pointer Declarators.
QualType getPointeeType() const
A (possibly-)qualified type.
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
LangAS getAddressSpace() const
Return the address space of this type.
QualType getCanonicalType() const
The collection of all-type qualifiers we support.
Flags to identify the types for overloaded SVE builtins.
bool isZExtReturn() const
bool isReverseUSDOT() const
bool isOverloadNone() const
MemEltType getMemEltType() const
bool isGatherLoad() const
bool isOverloadCvt() const
EltType getEltType() const
bool isOverloadDefault() const
bool isOverloadWhileRW() const
bool isInsertOp1SVALL() const
bool isOverloadWhile() const
bool isAppendSVALL() const
bool isStructStore() const
bool isTupleCreate() const
bool isGatherPrefetch() const
bool hasSplatOperand() const
MergeType getMergeType() const
bool isByteIndexed() const
bool isStructLoad() const
unsigned getSplatOperand() const
bool isScatterStore() const
bool isReverseCompare() const
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
SourceLocation getBeginLoc() const LLVM_READONLY
Exposes information about the current target.
TargetOptions & getTargetOpts() const
Retrieve the target options.
virtual bool hasLegalHalfType() const
Determine whether _Float16 is supported on this target.
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
bool isLittleEndian() const
virtual const char * getClobbers() const =0
Returns a string of target-specific clobbers, in LLVM format.
unsigned getMaxOpenCLWorkGroupSize() const
virtual bool checkArithmeticFenceSupported() const
Controls if __arithmetic_fence is supported in the targeted backend.
unsigned getSuitableAlign() const
Return the alignment that is the largest alignment ever used for any scalar/SIMD data type on the tar...
CodeObjectVersionKind CodeObjectVersion
Code object version for AMDGPU.
The base class of the type hierarchy.
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
bool isBlockPointerType() const
bool isBooleanType() const
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
bool isPointerType() const
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
const T * castAs() const
Member-template castAs<specific type>.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isBitIntType() const
bool isFloatingType() const
bool isUnsignedIntegerType() const
Return true if this is an integer type that is unsigned, according to C99 6.2.5p6 [which returns true...
const T * getAs() const
Member-template getAs<specific type>'.
Represents a GCC generic vector type.
SmallVector< OSLogBufferItem, 4 > Items
unsigned char getNumArgsByte() const
unsigned char getSummaryByte() const
Defines the clang::TargetInfo interface.
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
llvm::Constant * initializationPatternFor(CodeGenModule &, llvm::Type *)
TypeEvaluationKind
The kind of evaluation to perform on values of a particular type.
@ EHCleanup
Denotes a cleanup that should run when a scope is exited using exceptional control flow (a throw stat...
constexpr XRayInstrMask Typed
constexpr XRayInstrMask Custom
bool computeOSLogBufferLayout(clang::ASTContext &Ctx, const clang::CallExpr *E, OSLogBufferLayout &layout)
@ OS
Indicates that the tracking object is a descendant of a referenced-counted OSObject,...
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
bool Sub(InterpState &S, CodePtr OpPC)
bool Dup(InterpState &S, CodePtr OpPC)
bool Zero(InterpState &S, CodePtr OpPC)
unsigned Trunc(InterpState &S, CodePtr OpPC, unsigned Bits, const T &V)
bool Load(InterpState &S, CodePtr OpPC)
bool Cast(InterpState &S, CodePtr OpPC)
bool isa(CodeGen::Address addr)
@ OK_BitField
A bitfield object is a bitfield on a C or C++ record.
bool operator<(DeclarationName LHS, DeclarationName RHS)
Ordering on two declaration names.
@ C
Languages that the frontend can parse and compile.
@ Asm
Assembly: we accept this only so that we can preprocess it.
@ Result
The result type of a method or function.
U cast(CodeGen::Address addr)
YAML serialization mapping.
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::PointerType * Int8PtrPtrTy
CharUnits getIntAlign() const
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::Type * HalfTy
half, bfloat, float, double
llvm::IntegerType * SizeTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntPtrTy
llvm::IntegerType * IntTy
int
llvm::IntegerType * Int16Ty
llvm::PointerType * Int8PtrTy
llvm::PointerType * AllocaInt8PtrTy
EvalResult is a struct with detailed info about an evaluated expression.
APValue Val
Val - This is the value the expression can be folded to.
bool has(SanitizerMask K) const
Check if a certain (single) sanitizer is enabled.
bool has(XRayInstrMask K) const
#define scalbln(__x, __y)
#define copysign(__x, __y)
#define remquo(__x, __y, __z)
#define nextafter(__x, __y)
#define nexttoward(__x, __y)
#define remainder(__x, __y)
#define fma(__x, __y, __z)